mirror of https://github.com/Squidex/squidex.git
Sebastian
7 years ago
67 changed files with 2449 additions and 383 deletions
@ -1,147 +0,0 @@ |
|||||
// ==========================================================================
|
|
||||
// Squidex Headless CMS
|
|
||||
// ==========================================================================
|
|
||||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|
||||
// All rights reserved. Licensed under the MIT license.
|
|
||||
// ==========================================================================
|
|
||||
|
|
||||
using System; |
|
||||
using System.Collections.Generic; |
|
||||
using System.Linq; |
|
||||
using System.Threading; |
|
||||
using System.Threading.Tasks; |
|
||||
using MongoDB.Driver; |
|
||||
using NodaTime; |
|
||||
using Squidex.Domain.Apps.Core.ConvertContent; |
|
||||
using Squidex.Domain.Apps.Entities.Apps; |
|
||||
using Squidex.Domain.Apps.Entities.Contents; |
|
||||
using Squidex.Domain.Apps.Entities.Contents.State; |
|
||||
using Squidex.Domain.Apps.Entities.MongoDb.Contents.Visitors; |
|
||||
using Squidex.Domain.Apps.Entities.Schemas; |
|
||||
using Squidex.Infrastructure; |
|
||||
using Squidex.Infrastructure.Json; |
|
||||
using Squidex.Infrastructure.MongoDb; |
|
||||
using Squidex.Infrastructure.Queries; |
|
||||
using Squidex.Infrastructure.Reflection; |
|
||||
using Squidex.Infrastructure.States; |
|
||||
|
|
||||
namespace Squidex.Domain.Apps.Entities.MongoDb.Contents |
|
||||
{ |
|
||||
internal sealed class MongoContentDraftCollection : MongoContentCollection |
|
||||
{ |
|
||||
public MongoContentDraftCollection(IMongoDatabase database, IJsonSerializer serializer) |
|
||||
: base(database, serializer, "State_Content_Draft") |
|
||||
{ |
|
||||
} |
|
||||
|
|
||||
protected override async Task SetupCollectionAsync(IMongoCollection<MongoContentEntity> collection, CancellationToken ct = default) |
|
||||
{ |
|
||||
await collection.Indexes.CreateManyAsync( |
|
||||
new[] |
|
||||
{ |
|
||||
new CreateIndexModel<MongoContentEntity>( |
|
||||
Index |
|
||||
.Ascending(x => x.IndexedSchemaId) |
|
||||
.Ascending(x => x.Id) |
|
||||
.Ascending(x => x.IsDeleted)), |
|
||||
new CreateIndexModel<MongoContentEntity>( |
|
||||
Index |
|
||||
.Text(x => x.DataText) |
|
||||
.Ascending(x => x.IndexedSchemaId) |
|
||||
.Ascending(x => x.IsDeleted) |
|
||||
.Ascending(x => x.Status)) |
|
||||
}, ct); |
|
||||
|
|
||||
await base.SetupCollectionAsync(collection, ct); |
|
||||
} |
|
||||
|
|
||||
public async Task<IReadOnlyList<Guid>> QueryIdsAsync(Guid appId, ISchemaEntity schema, FilterNode filterNode) |
|
||||
{ |
|
||||
var filter = filterNode.AdjustToModel(schema.SchemaDef, true).ToFilter(schema.Id); |
|
||||
|
|
||||
var contentEntities = |
|
||||
await Collection.Find(filter).Only(x => x.Id) |
|
||||
.ToListAsync(); |
|
||||
|
|
||||
return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList(); |
|
||||
} |
|
||||
|
|
||||
public async Task<IReadOnlyList<Guid>> QueryIdsAsync(Guid appId) |
|
||||
{ |
|
||||
var contentEntities = |
|
||||
await Collection.Find(x => x.IndexedAppId == appId).Only(x => x.Id) |
|
||||
.ToListAsync(); |
|
||||
|
|
||||
return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList(); |
|
||||
} |
|
||||
|
|
||||
public Task QueryScheduledWithoutDataAsync(Instant now, Func<IContentEntity, Task> callback) |
|
||||
{ |
|
||||
return Collection.Find(x => x.ScheduledAt < now && x.IsDeleted != true) |
|
||||
.Not(x => x.DataByIds) |
|
||||
.Not(x => x.DataDraftByIds) |
|
||||
.Not(x => x.DataText) |
|
||||
.ForEachAsync(c => |
|
||||
{ |
|
||||
callback(c); |
|
||||
}); |
|
||||
} |
|
||||
|
|
||||
public async Task<IContentEntity> FindContentAsync(IAppEntity app, ISchemaEntity schema, Guid id) |
|
||||
{ |
|
||||
var contentEntity = |
|
||||
await Collection.Find(x => x.IndexedSchemaId == schema.Id && x.Id == id && x.IsDeleted != true).Not(x => x.DataText) |
|
||||
.FirstOrDefaultAsync(); |
|
||||
|
|
||||
contentEntity?.ParseData(schema.SchemaDef, Serializer); |
|
||||
|
|
||||
return contentEntity; |
|
||||
} |
|
||||
|
|
||||
public async Task<(ContentState Value, long Version)> ReadAsync(Guid key, Func<Guid, Guid, Task<ISchemaEntity>> getSchema) |
|
||||
{ |
|
||||
var contentEntity = |
|
||||
await Collection.Find(x => x.Id == key).Not(x => x.DataText) |
|
||||
.FirstOrDefaultAsync(); |
|
||||
|
|
||||
if (contentEntity != null) |
|
||||
{ |
|
||||
var schema = await getSchema(contentEntity.IndexedAppId, contentEntity.IndexedSchemaId); |
|
||||
|
|
||||
contentEntity.ParseData(schema.SchemaDef, Serializer); |
|
||||
|
|
||||
return (SimpleMapper.Map(contentEntity, new ContentState()), contentEntity.Version); |
|
||||
} |
|
||||
|
|
||||
return (null, EtagVersion.NotFound); |
|
||||
} |
|
||||
|
|
||||
public async Task UpsertAsync(MongoContentEntity content, long oldVersion) |
|
||||
{ |
|
||||
try |
|
||||
{ |
|
||||
content.DataText = content.DataDraftByIds.ToFullText(); |
|
||||
|
|
||||
await Collection.ReplaceOneAsync(x => x.Id == content.Id && x.Version == oldVersion, content, Upsert); |
|
||||
} |
|
||||
catch (MongoWriteException ex) |
|
||||
{ |
|
||||
if (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) |
|
||||
{ |
|
||||
var existingVersion = |
|
||||
await Collection.Find(x => x.Id == content.Id).Only(x => x.Id, x => x.Version) |
|
||||
.FirstOrDefaultAsync(); |
|
||||
|
|
||||
if (existingVersion != null) |
|
||||
{ |
|
||||
throw new InconsistentStateException(existingVersion["vs"].AsInt64, oldVersion, ex); |
|
||||
} |
|
||||
} |
|
||||
else |
|
||||
{ |
|
||||
throw; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -1,61 +0,0 @@ |
|||||
// ==========================================================================
|
|
||||
// Squidex Headless CMS
|
|
||||
// ==========================================================================
|
|
||||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|
||||
// All rights reserved. Licensed under the MIT license.
|
|
||||
// ==========================================================================
|
|
||||
|
|
||||
using System; |
|
||||
using System.Threading; |
|
||||
using System.Threading.Tasks; |
|
||||
using MongoDB.Driver; |
|
||||
using Squidex.Domain.Apps.Core.ConvertContent; |
|
||||
using Squidex.Domain.Apps.Entities.Apps; |
|
||||
using Squidex.Domain.Apps.Entities.Contents; |
|
||||
using Squidex.Domain.Apps.Entities.Schemas; |
|
||||
using Squidex.Infrastructure.Json; |
|
||||
using Squidex.Infrastructure.MongoDb; |
|
||||
|
|
||||
namespace Squidex.Domain.Apps.Entities.MongoDb.Contents |
|
||||
{ |
|
||||
internal sealed class MongoContentPublishedCollection : MongoContentCollection |
|
||||
{ |
|
||||
public MongoContentPublishedCollection(IMongoDatabase database, IJsonSerializer serializer) |
|
||||
: base(database, serializer, "State_Content_Published") |
|
||||
{ |
|
||||
} |
|
||||
|
|
||||
protected override async Task SetupCollectionAsync(IMongoCollection<MongoContentEntity> collection, CancellationToken ct = default) |
|
||||
{ |
|
||||
await collection.Indexes.CreateManyAsync( |
|
||||
new[] |
|
||||
{ |
|
||||
new CreateIndexModel<MongoContentEntity>(Index.Text(x => x.DataText).Ascending(x => x.IndexedSchemaId)), |
|
||||
new CreateIndexModel<MongoContentEntity>(Index.Ascending(x => x.IndexedSchemaId).Ascending(x => x.Id)) |
|
||||
}, ct); |
|
||||
|
|
||||
await base.SetupCollectionAsync(collection, ct); |
|
||||
} |
|
||||
|
|
||||
public async Task<IContentEntity> FindContentAsync(IAppEntity app, ISchemaEntity schema, Guid id) |
|
||||
{ |
|
||||
var contentEntity = |
|
||||
await Collection.Find(x => x.IndexedSchemaId == schema.Id && x.Id == id).Not(x => x.DataText) |
|
||||
.FirstOrDefaultAsync(); |
|
||||
|
|
||||
contentEntity?.ParseData(schema.SchemaDef, Serializer); |
|
||||
|
|
||||
return contentEntity; |
|
||||
} |
|
||||
|
|
||||
public Task UpsertAsync(MongoContentEntity content) |
|
||||
{ |
|
||||
content.DataText = content.DataByIds.ToFullText(); |
|
||||
content.DataDraftByIds = null; |
|
||||
content.ScheduleJob = null; |
|
||||
content.ScheduledAt = null; |
|
||||
|
|
||||
return Collection.ReplaceOneAsync(x => x.Id == content.Id, content, new UpdateOptions { IsUpsert = true }); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,106 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Linq; |
||||
|
using System.Threading.Tasks; |
||||
|
using Orleans; |
||||
|
using Squidex.Domain.Apps.Core.Contents; |
||||
|
using Squidex.Domain.Apps.Entities.Apps; |
||||
|
using Squidex.Domain.Apps.Entities.Schemas; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.Log; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed class GrainTextIndexer : ITextIndexer |
||||
|
{ |
||||
|
private readonly IGrainFactory grainFactory; |
||||
|
private readonly ISemanticLog log; |
||||
|
|
||||
|
public GrainTextIndexer(IGrainFactory grainFactory, ISemanticLog log) |
||||
|
{ |
||||
|
Guard.NotNull(grainFactory, nameof(grainFactory)); |
||||
|
Guard.NotNull(log, nameof(log)); |
||||
|
|
||||
|
this.grainFactory = grainFactory; |
||||
|
|
||||
|
this.log = log; |
||||
|
} |
||||
|
|
||||
|
public async Task DeleteAsync(Guid schemaId, Guid id) |
||||
|
{ |
||||
|
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); |
||||
|
|
||||
|
using (Profiler.TraceMethod<GrainTextIndexer>()) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
await index.DeleteAsync(id); |
||||
|
} |
||||
|
catch (Exception ex) |
||||
|
{ |
||||
|
log.LogError(ex, w => w |
||||
|
.WriteProperty("action", "DeleteTextEntry") |
||||
|
.WriteProperty("status", "Failed")); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft) |
||||
|
{ |
||||
|
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); |
||||
|
|
||||
|
using (Profiler.TraceMethod<GrainTextIndexer>()) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
if (data != null) |
||||
|
{ |
||||
|
await index.IndexAsync(id, new IndexData { Data = data }); |
||||
|
} |
||||
|
|
||||
|
if (dataDraft != null) |
||||
|
{ |
||||
|
await index.IndexAsync(id, new IndexData { Data = dataDraft, IsDraft = true }); |
||||
|
} |
||||
|
} |
||||
|
catch (Exception ex) |
||||
|
{ |
||||
|
log.LogError(ex, w => w |
||||
|
.WriteProperty("action", "UpdateTextEntry") |
||||
|
.WriteProperty("status", "Failed")); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false) |
||||
|
{ |
||||
|
if (string.IsNullOrWhiteSpace(queryText)) |
||||
|
{ |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); |
||||
|
|
||||
|
using (Profiler.TraceMethod<GrainTextIndexer>()) |
||||
|
{ |
||||
|
var context = CreateContext(app, useDraft); |
||||
|
|
||||
|
return await index.SearchAsync(queryText, context); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static SearchContext CreateContext(IAppEntity app, bool useDraft) |
||||
|
{ |
||||
|
var languages = new HashSet<string>(app.LanguagesConfig.Select(x => x.Key)); |
||||
|
|
||||
|
return new SearchContext { Languages = languages, IsDraft = useDraft }; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Threading.Tasks; |
||||
|
using Squidex.Domain.Apps.Core.Contents; |
||||
|
using Squidex.Domain.Apps.Entities.Apps; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public interface ITextIndexer |
||||
|
{ |
||||
|
Task DeleteAsync(Guid schemaId, Guid id); |
||||
|
|
||||
|
Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft); |
||||
|
|
||||
|
Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Threading.Tasks; |
||||
|
using Orleans; |
||||
|
using Squidex.Infrastructure.Orleans; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public interface ITextIndexerGrain : IGrainWithGuidKey |
||||
|
{ |
||||
|
Task DeleteAsync(Guid id); |
||||
|
|
||||
|
Task IndexAsync(Guid id, J<IndexData> data); |
||||
|
|
||||
|
Task<List<Guid>> SearchAsync(string queryText, SearchContext context); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,18 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using Squidex.Domain.Apps.Core.Contents; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed class IndexData |
||||
|
{ |
||||
|
public NamedContentData Data { get; set; } |
||||
|
|
||||
|
public bool IsDraft { get; set; } |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,65 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Linq; |
||||
|
using Lucene.Net.Analysis; |
||||
|
using Lucene.Net.Analysis.Standard; |
||||
|
using Lucene.Net.Util; |
||||
|
using Squidex.Infrastructure; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed class MultiLanguageAnalyzer : AnalyzerWrapper |
||||
|
{ |
||||
|
private readonly StandardAnalyzer fallbackAnalyzer; |
||||
|
private readonly Dictionary<string, Analyzer> analyzers = new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase); |
||||
|
|
||||
|
public MultiLanguageAnalyzer(LuceneVersion version) |
||||
|
: base(PER_FIELD_REUSE_STRATEGY) |
||||
|
{ |
||||
|
fallbackAnalyzer = new StandardAnalyzer(version); |
||||
|
|
||||
|
foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) |
||||
|
{ |
||||
|
if (typeof(Analyzer).IsAssignableFrom(type)) |
||||
|
{ |
||||
|
var language = type.Namespace.Split('.').Last(); |
||||
|
|
||||
|
if (language.Length == 2) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
var analyzer = Activator.CreateInstance(type, version); |
||||
|
|
||||
|
analyzers[language] = (Analyzer)analyzer; |
||||
|
} |
||||
|
catch (MissingMethodException) |
||||
|
{ |
||||
|
continue; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
protected override Analyzer GetWrappedAnalyzer(string fieldName) |
||||
|
{ |
||||
|
if (fieldName.Length > 0) |
||||
|
{ |
||||
|
var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer; |
||||
|
|
||||
|
return analyzer; |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
return fallbackAnalyzer; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,91 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.IO; |
||||
|
using System.IO.Compression; |
||||
|
using System.Threading.Tasks; |
||||
|
using Squidex.Infrastructure.Assets; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public static class PersistenceHelper |
||||
|
{ |
||||
|
private const string ArchiveFile = "Archive.zip"; |
||||
|
private const string LockFile = "write.lock"; |
||||
|
|
||||
|
public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory) |
||||
|
{ |
||||
|
using (var fileStream = new FileStream( |
||||
|
Path.Combine(directory.FullName, ArchiveFile), |
||||
|
FileMode.Create, |
||||
|
FileAccess.ReadWrite, |
||||
|
FileShare.None, |
||||
|
4096, |
||||
|
FileOptions.DeleteOnClose)) |
||||
|
{ |
||||
|
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) |
||||
|
{ |
||||
|
foreach (var file in directory.GetFiles()) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && |
||||
|
!file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) |
||||
|
{ |
||||
|
zipArchive.CreateEntryFromFile(file.FullName, file.Name); |
||||
|
} |
||||
|
} |
||||
|
catch (IOException) |
||||
|
{ |
||||
|
continue; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
fileStream.Position = 0; |
||||
|
|
||||
|
await assetStore.UploadAsync(directory.Name, 0, string.Empty, fileStream, true); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static async Task DownloadAsync(this IAssetStore assetStore, DirectoryInfo directory) |
||||
|
{ |
||||
|
if (directory.Exists) |
||||
|
{ |
||||
|
directory.Delete(true); |
||||
|
} |
||||
|
|
||||
|
directory.Create(); |
||||
|
|
||||
|
using (var fileStream = new FileStream( |
||||
|
Path.Combine(directory.FullName, ArchiveFile), |
||||
|
FileMode.Create, |
||||
|
FileAccess.ReadWrite, |
||||
|
FileShare.None, |
||||
|
4096, |
||||
|
FileOptions.DeleteOnClose)) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
await assetStore.DownloadAsync(directory.Name, 0, string.Empty, fileStream); |
||||
|
|
||||
|
fileStream.Position = 0; |
||||
|
|
||||
|
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) |
||||
|
{ |
||||
|
zipArchive.ExtractToDirectory(directory.FullName); |
||||
|
} |
||||
|
} |
||||
|
catch (AssetNotFoundException) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,18 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Collections.Generic; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed class SearchContext |
||||
|
{ |
||||
|
public bool IsDraft { get; set; } |
||||
|
|
||||
|
public HashSet<string> Languages { get; set; } |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,272 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.IO; |
||||
|
using System.Linq; |
||||
|
using System.Text; |
||||
|
using System.Threading.Tasks; |
||||
|
using Lucene.Net.Analysis; |
||||
|
using Lucene.Net.Documents; |
||||
|
using Lucene.Net.Index; |
||||
|
using Lucene.Net.Queries; |
||||
|
using Lucene.Net.QueryParsers.Classic; |
||||
|
using Lucene.Net.Search; |
||||
|
using Lucene.Net.Store; |
||||
|
using Lucene.Net.Util; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.Assets; |
||||
|
using Squidex.Infrastructure.Json.Objects; |
||||
|
using Squidex.Infrastructure.Orleans; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed class TextIndexerGrain : GrainOfGuid, ITextIndexerGrain |
||||
|
{ |
||||
|
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
||||
|
private const int MaxResults = 2000; |
||||
|
private const int MaxUpdates = 100; |
||||
|
private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30); |
||||
|
private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); |
||||
|
private readonly IAssetStore assetStore; |
||||
|
private IDisposable timer; |
||||
|
private DirectoryInfo directory; |
||||
|
private IndexWriter indexWriter; |
||||
|
private IndexReader indexReader; |
||||
|
private QueryParser queryParser; |
||||
|
private HashSet<string> currentLanguages; |
||||
|
private long updates; |
||||
|
|
||||
|
public TextIndexerGrain(IAssetStore assetStore) |
||||
|
{ |
||||
|
Guard.NotNull(assetStore, nameof(assetStore)); |
||||
|
|
||||
|
this.assetStore = assetStore; |
||||
|
} |
||||
|
|
||||
|
public override async Task OnDeactivateAsync() |
||||
|
{ |
||||
|
await DeactivateAsync(true); |
||||
|
} |
||||
|
|
||||
|
protected override async Task OnActivateAsync(Guid key) |
||||
|
{ |
||||
|
directory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), $"Index_{key}")); |
||||
|
|
||||
|
await assetStore.DownloadAsync(directory); |
||||
|
|
||||
|
indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer)); |
||||
|
indexReader = indexWriter.GetReader(true); |
||||
|
} |
||||
|
|
||||
|
public Task DeleteAsync(Guid id) |
||||
|
{ |
||||
|
indexWriter.DeleteDocuments(new Term("id", id.ToString())); |
||||
|
|
||||
|
return TryFlushAsync(); |
||||
|
} |
||||
|
|
||||
|
public Task IndexAsync(Guid id, J<IndexData> data) |
||||
|
{ |
||||
|
var docId = id.ToString(); |
||||
|
var docDraft = data.Value.IsDraft.ToString(); |
||||
|
var docKey = $"{docId}_{docDraft}"; |
||||
|
|
||||
|
var query = new BooleanQuery(); |
||||
|
|
||||
|
indexWriter.DeleteDocuments(new Term("key", docKey)); |
||||
|
|
||||
|
var languages = new Dictionary<string, StringBuilder>(); |
||||
|
|
||||
|
void AppendText(string language, string text) |
||||
|
{ |
||||
|
if (!string.IsNullOrWhiteSpace(text)) |
||||
|
{ |
||||
|
var sb = languages.GetOrAddNew(language); |
||||
|
|
||||
|
if (sb.Length > 0) |
||||
|
{ |
||||
|
sb.Append(" "); |
||||
|
} |
||||
|
|
||||
|
sb.Append(text); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
foreach (var field in data.Value.Data) |
||||
|
{ |
||||
|
foreach (var fieldValue in field.Value) |
||||
|
{ |
||||
|
var appendText = new Action<string>(text => AppendText(fieldValue.Key, text)); |
||||
|
|
||||
|
AppendJsonText(fieldValue.Value, appendText); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (languages.Count > 0) |
||||
|
{ |
||||
|
var document = new Document(); |
||||
|
|
||||
|
document.AddStringField("id", docId, Field.Store.YES); |
||||
|
document.AddStringField("key", docKey, Field.Store.YES); |
||||
|
document.AddStringField("draft", docDraft, Field.Store.YES); |
||||
|
|
||||
|
foreach (var field in languages) |
||||
|
{ |
||||
|
var fieldName = BuildFieldName(field.Key); |
||||
|
|
||||
|
document.AddTextField(fieldName, field.Value.ToString(), Field.Store.NO); |
||||
|
} |
||||
|
|
||||
|
indexWriter.AddDocument(document); |
||||
|
} |
||||
|
|
||||
|
return TryFlushAsync(); |
||||
|
} |
||||
|
|
||||
|
private static void AppendJsonText(IJsonValue value, Action<string> appendText) |
||||
|
{ |
||||
|
if (value.Type == JsonValueType.String) |
||||
|
{ |
||||
|
appendText(value.ToString()); |
||||
|
} |
||||
|
else if (value is JsonArray array) |
||||
|
{ |
||||
|
foreach (var item in array) |
||||
|
{ |
||||
|
AppendJsonText(item, appendText); |
||||
|
} |
||||
|
} |
||||
|
else if (value is JsonObject obj) |
||||
|
{ |
||||
|
foreach (var item in obj.Values) |
||||
|
{ |
||||
|
AppendJsonText(item, appendText); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public Task<List<Guid>> SearchAsync(string queryText, SearchContext context) |
||||
|
{ |
||||
|
var result = new HashSet<Guid>(); |
||||
|
|
||||
|
if (!string.IsNullOrWhiteSpace(queryText)) |
||||
|
{ |
||||
|
var query = BuildQuery(queryText, context); |
||||
|
|
||||
|
if (indexReader != null) |
||||
|
{ |
||||
|
var filter = new TermsFilter(new Term("draft", context.IsDraft.ToString())); |
||||
|
|
||||
|
var hits = new IndexSearcher(indexReader).Search(query, filter, MaxResults).ScoreDocs; |
||||
|
|
||||
|
foreach (var hit in hits) |
||||
|
{ |
||||
|
var document = indexReader.Document(hit.Doc); |
||||
|
|
||||
|
var idField = document.GetField("id")?.GetStringValue(); |
||||
|
|
||||
|
if (idField != null && Guid.TryParse(idField, out var guid)) |
||||
|
{ |
||||
|
result.Add(guid); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return Task.FromResult(result.ToList()); |
||||
|
} |
||||
|
|
||||
|
private Query BuildQuery(string query, SearchContext context) |
||||
|
{ |
||||
|
if (queryParser == null || !currentLanguages.SetEquals(context.Languages)) |
||||
|
{ |
||||
|
var fields = |
||||
|
context.Languages.Select(BuildFieldName) |
||||
|
.Union(Enumerable.Repeat(BuildFieldName("iv"), 1)).ToArray(); |
||||
|
|
||||
|
queryParser = new MultiFieldQueryParser(Version, fields, Analyzer); |
||||
|
|
||||
|
currentLanguages = context.Languages; |
||||
|
} |
||||
|
|
||||
|
try |
||||
|
{ |
||||
|
return queryParser.Parse(query); |
||||
|
} |
||||
|
catch (ParseException ex) |
||||
|
{ |
||||
|
throw new ValidationException(ex.Message); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private async Task TryFlushAsync() |
||||
|
{ |
||||
|
updates++; |
||||
|
|
||||
|
if (updates >= MaxUpdates) |
||||
|
{ |
||||
|
await FlushAsync(); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
timer?.Dispose(); |
||||
|
|
||||
|
try |
||||
|
{ |
||||
|
timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay); |
||||
|
} |
||||
|
catch (InvalidOperationException) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task FlushAsync() |
||||
|
{ |
||||
|
if (updates > 0 && indexWriter != null) |
||||
|
{ |
||||
|
indexWriter.Flush(true, true); |
||||
|
indexWriter.Commit(); |
||||
|
|
||||
|
indexReader?.Dispose(); |
||||
|
indexReader = indexWriter.GetReader(true); |
||||
|
|
||||
|
await assetStore.UploadDirectoryAsync(directory); |
||||
|
|
||||
|
updates = 0; |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
timer?.Dispose(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task DeactivateAsync(bool deleteFolder = false) |
||||
|
{ |
||||
|
await TryFlushAsync(); |
||||
|
|
||||
|
indexWriter?.Dispose(); |
||||
|
indexWriter = null; |
||||
|
|
||||
|
indexReader?.Dispose(); |
||||
|
indexReader = null; |
||||
|
|
||||
|
if (deleteFolder && directory.Exists) |
||||
|
{ |
||||
|
directory.Delete(true); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static string BuildFieldName(string language) |
||||
|
{ |
||||
|
return $"{language}_field"; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,32 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Azure.Documents.Client; |
||||
|
using Microsoft.Extensions.Diagnostics.HealthChecks; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.Diagnostics |
||||
|
{ |
||||
|
public sealed class CosmosDbHealthCheck : IHealthCheck |
||||
|
{ |
||||
|
private readonly DocumentClient documentClient; |
||||
|
|
||||
|
public CosmosDbHealthCheck(Uri uri, string masterKey) |
||||
|
{ |
||||
|
documentClient = new DocumentClient(uri, masterKey); |
||||
|
} |
||||
|
|
||||
|
public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) |
||||
|
{ |
||||
|
await documentClient.ReadDatabaseFeedAsync(); |
||||
|
|
||||
|
return HealthCheckResult.Healthy("Application must query data from CosmosDB."); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,16 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal static class Constants |
||||
|
{ |
||||
|
public const string Collection = "Events"; |
||||
|
|
||||
|
public const string LeaseCollection = "Leases"; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,33 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using Newtonsoft.Json; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal sealed class CosmosDbEvent |
||||
|
{ |
||||
|
[JsonProperty("type")] |
||||
|
public string Type { get; set; } |
||||
|
|
||||
|
[JsonProperty("payload")] |
||||
|
public string Payload { get; set; } |
||||
|
|
||||
|
[JsonProperty("header")] |
||||
|
public EnvelopeHeaders Headers { get; set; } |
||||
|
|
||||
|
public static CosmosDbEvent FromEventData(EventData data) |
||||
|
{ |
||||
|
return new CosmosDbEvent { Type = data.Type, Headers = data.Headers, Payload = data.Payload }; |
||||
|
} |
||||
|
|
||||
|
public EventData ToEventData() |
||||
|
{ |
||||
|
return new EventData(Type, Headers, Payload); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,33 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using Newtonsoft.Json; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal sealed class CosmosDbEventCommit |
||||
|
{ |
||||
|
[JsonProperty("id")] |
||||
|
public Guid Id { get; set; } |
||||
|
|
||||
|
[JsonProperty("events")] |
||||
|
public CosmosDbEvent[] Events { get; set; } |
||||
|
|
||||
|
[JsonProperty("eventStreamOffset")] |
||||
|
public long EventStreamOffset { get; set; } |
||||
|
|
||||
|
[JsonProperty("eventsCount")] |
||||
|
public long EventsCount { get; set; } |
||||
|
|
||||
|
[JsonProperty("eventStream")] |
||||
|
public string EventStream { get; set; } |
||||
|
|
||||
|
[JsonProperty("timestamp")] |
||||
|
public long Timestamp { get; set; } |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,124 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.ObjectModel; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Azure.Documents; |
||||
|
using Microsoft.Azure.Documents.Client; |
||||
|
using Newtonsoft.Json; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
public sealed partial class CosmosDbEventStore : DisposableObjectBase, IEventStore, IInitializable |
||||
|
{ |
||||
|
private readonly DocumentClient documentClient; |
||||
|
private readonly Uri collectionUri; |
||||
|
private readonly Uri databaseUri; |
||||
|
private readonly string masterKey; |
||||
|
private readonly string databaseId; |
||||
|
private readonly JsonSerializerSettings serializerSettings; |
||||
|
|
||||
|
public JsonSerializerSettings SerializerSettings |
||||
|
{ |
||||
|
get { return serializerSettings; } |
||||
|
} |
||||
|
|
||||
|
public string DatabaseId |
||||
|
{ |
||||
|
get { return databaseId; } |
||||
|
} |
||||
|
|
||||
|
public string MasterKey |
||||
|
{ |
||||
|
get { return masterKey; } |
||||
|
} |
||||
|
|
||||
|
public Uri ServiceUri |
||||
|
{ |
||||
|
get { return documentClient.ServiceEndpoint; } |
||||
|
} |
||||
|
|
||||
|
public CosmosDbEventStore(DocumentClient documentClient, string masterKey, string database, JsonSerializerSettings serializerSettings) |
||||
|
{ |
||||
|
Guard.NotNull(documentClient, nameof(documentClient)); |
||||
|
Guard.NotNull(serializerSettings, nameof(serializerSettings)); |
||||
|
Guard.NotNullOrEmpty(masterKey, nameof(masterKey)); |
||||
|
Guard.NotNullOrEmpty(database, nameof(database)); |
||||
|
|
||||
|
this.documentClient = documentClient; |
||||
|
|
||||
|
databaseUri = UriFactory.CreateDatabaseUri(database); |
||||
|
databaseId = database; |
||||
|
|
||||
|
collectionUri = UriFactory.CreateDocumentCollectionUri(database, Constants.Collection); |
||||
|
|
||||
|
this.masterKey = masterKey; |
||||
|
|
||||
|
this.serializerSettings = serializerSettings; |
||||
|
} |
||||
|
|
||||
|
protected override void DisposeObject(bool disposing) |
||||
|
{ |
||||
|
if (disposing) |
||||
|
{ |
||||
|
documentClient.Dispose(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task InitializeAsync(CancellationToken ct = default) |
||||
|
{ |
||||
|
await documentClient.CreateDatabaseIfNotExistsAsync(new Database { Id = databaseId }); |
||||
|
|
||||
|
await documentClient.CreateDocumentCollectionIfNotExistsAsync(databaseUri, |
||||
|
new DocumentCollection |
||||
|
{ |
||||
|
Id = Constants.LeaseCollection, |
||||
|
}); |
||||
|
|
||||
|
await documentClient.CreateDocumentCollectionIfNotExistsAsync(databaseUri, |
||||
|
new DocumentCollection |
||||
|
{ |
||||
|
IndexingPolicy = new IndexingPolicy |
||||
|
{ |
||||
|
IncludedPaths = new Collection<IncludedPath> |
||||
|
{ |
||||
|
new IncludedPath |
||||
|
{ |
||||
|
Path = "/*", |
||||
|
Indexes = new Collection<Index> |
||||
|
{ |
||||
|
Index.Range(DataType.Number), |
||||
|
Index.Range(DataType.String), |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}, |
||||
|
UniqueKeyPolicy = new UniqueKeyPolicy |
||||
|
{ |
||||
|
UniqueKeys = new Collection<UniqueKey> |
||||
|
{ |
||||
|
new UniqueKey |
||||
|
{ |
||||
|
Paths = new Collection<string> |
||||
|
{ |
||||
|
$"/eventStream", |
||||
|
$"/eventStreamOffset" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}, |
||||
|
Id = Constants.Collection, |
||||
|
}, |
||||
|
new RequestOptions |
||||
|
{ |
||||
|
PartitionKey = new PartitionKey($"/eventStream") |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,142 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Azure.Documents; |
||||
|
using Squidex.Infrastructure.Log; |
||||
|
using Squidex.Infrastructure.Tasks; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
public delegate bool EventPredicate(EventData data); |
||||
|
|
||||
|
public partial class CosmosDbEventStore : IEventStore, IInitializable |
||||
|
{ |
||||
|
public IEventSubscription CreateSubscription(IEventSubscriber subscriber, string streamFilter = null, string position = null) |
||||
|
{ |
||||
|
Guard.NotNull(subscriber, nameof(subscriber)); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
return new CosmosDbSubscription(this, subscriber, streamFilter, position); |
||||
|
} |
||||
|
|
||||
|
public Task CreateIndexAsync(string property) |
||||
|
{ |
||||
|
Guard.NotNullOrEmpty(property, nameof(property)); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
return Task.CompletedTask; |
||||
|
} |
||||
|
|
||||
|
public async Task<IReadOnlyList<StoredEvent>> QueryAsync(string streamName, long streamPosition = 0) |
||||
|
{ |
||||
|
Guard.NotNullOrEmpty(streamName, nameof(streamName)); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
using (Profiler.TraceMethod<CosmosDbEventStore>()) |
||||
|
{ |
||||
|
var query = FilterBuilder.ByStreamName(streamName, streamPosition - MaxCommitSize); |
||||
|
|
||||
|
var result = new List<StoredEvent>(); |
||||
|
|
||||
|
await documentClient.QueryAsync(collectionUri, query, commit => |
||||
|
{ |
||||
|
var eventStreamOffset = (int)commit.EventStreamOffset; |
||||
|
|
||||
|
var commitTimestamp = commit.Timestamp; |
||||
|
var commitOffset = 0; |
||||
|
|
||||
|
foreach (var @event in commit.Events) |
||||
|
{ |
||||
|
eventStreamOffset++; |
||||
|
|
||||
|
if (eventStreamOffset >= streamPosition) |
||||
|
{ |
||||
|
var eventData = @event.ToEventData(); |
||||
|
var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); |
||||
|
|
||||
|
result.Add(new StoredEvent(streamName, eventToken, eventStreamOffset, eventData)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return TaskHelper.Done; |
||||
|
}); |
||||
|
|
||||
|
return result; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public Task QueryAsync(Func<StoredEvent, Task> callback, string property, object value, string position = null, CancellationToken ct = default) |
||||
|
{ |
||||
|
Guard.NotNull(callback, nameof(callback)); |
||||
|
Guard.NotNullOrEmpty(property, nameof(property)); |
||||
|
Guard.NotNull(value, nameof(value)); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
StreamPosition lastPosition = position; |
||||
|
|
||||
|
var filterDefinition = FilterBuilder.CreateByProperty(property, value, lastPosition); |
||||
|
var filterExpression = FilterBuilder.CreateExpression(property, value); |
||||
|
|
||||
|
return QueryAsync(callback, lastPosition, filterDefinition, filterExpression, ct); |
||||
|
} |
||||
|
|
||||
|
public Task QueryAsync(Func<StoredEvent, Task> callback, string streamFilter = null, string position = null, CancellationToken ct = default) |
||||
|
{ |
||||
|
Guard.NotNull(callback, nameof(callback)); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
StreamPosition lastPosition = position; |
||||
|
|
||||
|
var filterDefinition = FilterBuilder.CreateByFilter(streamFilter, lastPosition); |
||||
|
var filterExpression = FilterBuilder.CreateExpression(null, null); |
||||
|
|
||||
|
return QueryAsync(callback, lastPosition, filterDefinition, filterExpression, ct); |
||||
|
} |
||||
|
|
||||
|
private async Task QueryAsync(Func<StoredEvent, Task> callback, StreamPosition lastPosition, SqlQuerySpec query, EventPredicate filterExpression, CancellationToken ct = default) |
||||
|
{ |
||||
|
using (Profiler.TraceMethod<CosmosDbEventStore>()) |
||||
|
{ |
||||
|
await documentClient.QueryAsync(collectionUri, query, async commit => |
||||
|
{ |
||||
|
var eventStreamOffset = (int)commit.EventStreamOffset; |
||||
|
|
||||
|
var commitTimestamp = commit.Timestamp; |
||||
|
var commitOffset = 0; |
||||
|
|
||||
|
foreach (var @event in commit.Events) |
||||
|
{ |
||||
|
eventStreamOffset++; |
||||
|
|
||||
|
if (commitOffset > lastPosition.CommitOffset || commitTimestamp > lastPosition.Timestamp) |
||||
|
{ |
||||
|
var eventData = @event.ToEventData(); |
||||
|
|
||||
|
if (filterExpression(eventData)) |
||||
|
{ |
||||
|
var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); |
||||
|
|
||||
|
await callback(new StoredEvent(commit.EventStream, eventToken, eventStreamOffset, eventData)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
commitOffset++; |
||||
|
} |
||||
|
}, ct); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,149 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Net; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Azure.Documents; |
||||
|
using Microsoft.Azure.Documents.Client; |
||||
|
using NodaTime; |
||||
|
using Squidex.Infrastructure.Log; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
public partial class CosmosDbEventStore |
||||
|
{ |
||||
|
private const int MaxWriteAttempts = 20; |
||||
|
private const int MaxCommitSize = 10; |
||||
|
|
||||
|
public Task DeleteStreamAsync(string streamName) |
||||
|
{ |
||||
|
Guard.NotNullOrEmpty(streamName, nameof(streamName)); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
var query = FilterBuilder.AllIds(streamName); |
||||
|
|
||||
|
return documentClient.QueryAsync(collectionUri, query, commit => |
||||
|
{ |
||||
|
var documentUri = UriFactory.CreateDocumentUri(databaseId, Constants.Collection, commit.Id.ToString()); |
||||
|
|
||||
|
return documentClient.DeleteDocumentAsync(documentUri); |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
public Task AppendAsync(Guid commitId, string streamName, ICollection<EventData> events) |
||||
|
{ |
||||
|
return AppendAsync(commitId, streamName, EtagVersion.Any, events); |
||||
|
} |
||||
|
|
||||
|
public async Task AppendAsync(Guid commitId, string streamName, long expectedVersion, ICollection<EventData> events) |
||||
|
{ |
||||
|
Guard.NotEmpty(commitId, nameof(commitId)); |
||||
|
Guard.NotNullOrEmpty(streamName, nameof(streamName)); |
||||
|
Guard.NotNull(events, nameof(events)); |
||||
|
Guard.LessThan(events.Count, MaxCommitSize, "events.Count"); |
||||
|
|
||||
|
ThrowIfDisposed(); |
||||
|
|
||||
|
using (Profiler.TraceMethod<CosmosDbEventStore>()) |
||||
|
{ |
||||
|
if (events.Count == 0) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
var currentVersion = await GetEventStreamOffsetAsync(streamName); |
||||
|
|
||||
|
if (expectedVersion != EtagVersion.Any && expectedVersion != currentVersion) |
||||
|
{ |
||||
|
throw new WrongEventVersionException(currentVersion, expectedVersion); |
||||
|
} |
||||
|
|
||||
|
var commit = BuildCommit(commitId, streamName, expectedVersion >= -1 ? expectedVersion : currentVersion, events); |
||||
|
|
||||
|
for (var attempt = 0; attempt < MaxWriteAttempts; attempt++) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
await documentClient.CreateDocumentAsync(collectionUri, commit); |
||||
|
|
||||
|
return; |
||||
|
} |
||||
|
catch (DocumentClientException ex) |
||||
|
{ |
||||
|
if (ex.StatusCode == HttpStatusCode.Conflict) |
||||
|
{ |
||||
|
currentVersion = await GetEventStreamOffsetAsync(streamName); |
||||
|
|
||||
|
if (expectedVersion != EtagVersion.Any) |
||||
|
{ |
||||
|
throw new WrongEventVersionException(currentVersion, expectedVersion); |
||||
|
} |
||||
|
|
||||
|
if (attempt < MaxWriteAttempts) |
||||
|
{ |
||||
|
expectedVersion = currentVersion; |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
throw new TimeoutException("Could not acquire a free slot for the commit within the provided time."); |
||||
|
} |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
throw; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private async Task<long> GetEventStreamOffsetAsync(string streamName) |
||||
|
{ |
||||
|
var query = |
||||
|
documentClient.CreateDocumentQuery<CosmosDbEventCommit>(collectionUri, |
||||
|
FilterBuilder.LastPosition(streamName)); |
||||
|
|
||||
|
var document = await query.FirstOrDefaultAsync(); |
||||
|
|
||||
|
if (document != null) |
||||
|
{ |
||||
|
return document.EventStreamOffset + document.EventsCount; |
||||
|
} |
||||
|
|
||||
|
return EtagVersion.Empty; |
||||
|
} |
||||
|
|
||||
|
private static CosmosDbEventCommit BuildCommit(Guid commitId, string streamName, long expectedVersion, ICollection<EventData> events) |
||||
|
{ |
||||
|
var commitEvents = new CosmosDbEvent[events.Count]; |
||||
|
|
||||
|
var i = 0; |
||||
|
|
||||
|
foreach (var e in events) |
||||
|
{ |
||||
|
var mongoEvent = CosmosDbEvent.FromEventData(e); |
||||
|
|
||||
|
commitEvents[i++] = mongoEvent; |
||||
|
} |
||||
|
|
||||
|
var mongoCommit = new CosmosDbEventCommit |
||||
|
{ |
||||
|
Id = commitId, |
||||
|
Events = commitEvents, |
||||
|
EventsCount = events.Count, |
||||
|
EventStream = streamName, |
||||
|
EventStreamOffset = expectedVersion, |
||||
|
Timestamp = SystemClock.Instance.GetCurrentInstant().ToUnixTimeTicks() |
||||
|
}; |
||||
|
|
||||
|
return mongoCommit; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,150 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Text.RegularExpressions; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Azure.Documents; |
||||
|
using Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing; |
||||
|
using Newtonsoft.Json; |
||||
|
using Builder = Microsoft.Azure.Documents.ChangeFeedProcessor.ChangeFeedProcessorBuilder; |
||||
|
using Collection = Microsoft.Azure.Documents.ChangeFeedProcessor.DocumentCollectionInfo; |
||||
|
using Options = Microsoft.Azure.Documents.ChangeFeedProcessor.ChangeFeedProcessorOptions; |
||||
|
|
||||
|
#pragma warning disable IDE0017 // Simplify object initialization
|
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal sealed class CosmosDbSubscription : IEventSubscription, IChangeFeedObserverFactory, IChangeFeedObserver |
||||
|
{ |
||||
|
private readonly TaskCompletionSource<bool> processorStopRequested = new TaskCompletionSource<bool>(); |
||||
|
private readonly Task processorTask; |
||||
|
private readonly CosmosDbEventStore store; |
||||
|
private readonly Regex regex; |
||||
|
private readonly string hostName; |
||||
|
private readonly IEventSubscriber subscriber; |
||||
|
|
||||
|
public CosmosDbSubscription(CosmosDbEventStore store, IEventSubscriber subscriber, string streamFilter, string position = null) |
||||
|
{ |
||||
|
this.store = store; |
||||
|
|
||||
|
var fromBeginning = string.IsNullOrWhiteSpace(position); |
||||
|
|
||||
|
if (fromBeginning) |
||||
|
{ |
||||
|
hostName = $"squidex.{DateTime.UtcNow.Ticks.ToString()}"; |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
hostName = position; |
||||
|
} |
||||
|
|
||||
|
if (!StreamFilter.IsAll(streamFilter)) |
||||
|
{ |
||||
|
regex = new Regex(streamFilter); |
||||
|
} |
||||
|
|
||||
|
this.subscriber = subscriber; |
||||
|
|
||||
|
processorTask = Task.Run(async () => |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
Collection CreateCollection(string name) |
||||
|
{ |
||||
|
var collection = new Collection(); |
||||
|
|
||||
|
collection.CollectionName = name; |
||||
|
collection.DatabaseName = store.DatabaseId; |
||||
|
collection.MasterKey = store.MasterKey; |
||||
|
collection.Uri = store.ServiceUri; |
||||
|
|
||||
|
return collection; |
||||
|
} |
||||
|
|
||||
|
var processor = |
||||
|
await new Builder() |
||||
|
.WithFeedCollection(CreateCollection(Constants.Collection)) |
||||
|
.WithLeaseCollection(CreateCollection(Constants.LeaseCollection)) |
||||
|
.WithHostName(hostName) |
||||
|
.WithProcessorOptions(new Options { StartFromBeginning = fromBeginning, LeasePrefix = hostName }) |
||||
|
.WithObserverFactory(this) |
||||
|
.BuildAsync(); |
||||
|
|
||||
|
await processor.StartAsync(); |
||||
|
await processorStopRequested.Task; |
||||
|
await processor.StopAsync(); |
||||
|
} |
||||
|
catch (Exception ex) |
||||
|
{ |
||||
|
await subscriber.OnErrorAsync(this, ex); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
public IChangeFeedObserver CreateObserver() |
||||
|
{ |
||||
|
return this; |
||||
|
} |
||||
|
|
||||
|
public async Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason) |
||||
|
{ |
||||
|
if (reason == ChangeFeedObserverCloseReason.ObserverError) |
||||
|
{ |
||||
|
await subscriber.OnErrorAsync(this, new InvalidOperationException("Change feed observer failed.")); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public Task OpenAsync(IChangeFeedObserverContext context) |
||||
|
{ |
||||
|
return Task.CompletedTask; |
||||
|
} |
||||
|
|
||||
|
public async Task ProcessChangesAsync(IChangeFeedObserverContext context, IReadOnlyList<Document> docs, CancellationToken cancellationToken) |
||||
|
{ |
||||
|
if (!processorStopRequested.Task.IsCompleted) |
||||
|
{ |
||||
|
foreach (var document in docs) |
||||
|
{ |
||||
|
if (!processorStopRequested.Task.IsCompleted) |
||||
|
{ |
||||
|
var streamName = document.GetPropertyValue<string>("eventStream"); |
||||
|
|
||||
|
if (regex == null || regex.IsMatch(streamName)) |
||||
|
{ |
||||
|
var commit = JsonConvert.DeserializeObject<CosmosDbEventCommit>(document.ToString(), store.SerializerSettings); |
||||
|
|
||||
|
var eventStreamOffset = (int)commit.EventStreamOffset; |
||||
|
|
||||
|
foreach (var @event in commit.Events) |
||||
|
{ |
||||
|
eventStreamOffset++; |
||||
|
|
||||
|
var eventData = @event.ToEventData(); |
||||
|
|
||||
|
await subscriber.OnEventAsync(this, new StoredEvent(commit.EventStream, hostName, eventStreamOffset, eventData)); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void WakeUp() |
||||
|
{ |
||||
|
} |
||||
|
|
||||
|
public Task StopAsync() |
||||
|
{ |
||||
|
processorStopRequested.SetResult(true); |
||||
|
|
||||
|
return processorTask; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,156 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Collections.Generic; |
||||
|
using Microsoft.Azure.Documents; |
||||
|
using Squidex.Infrastructure.Json.Objects; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal static class FilterBuilder |
||||
|
{ |
||||
|
public static SqlQuerySpec AllIds(string streamName) |
||||
|
{ |
||||
|
var query = |
||||
|
$"SELECT TOP 1 " + |
||||
|
$" e.id," + |
||||
|
$" e.eventsCount " + |
||||
|
$"FROM {Constants.Collection} e " + |
||||
|
$"WHERE " + |
||||
|
$" e.eventStream = @name " + |
||||
|
$"ORDER BY e.eventStreamOffset DESC"; |
||||
|
|
||||
|
var parameters = new SqlParameterCollection |
||||
|
{ |
||||
|
new SqlParameter("@name", streamName) |
||||
|
}; |
||||
|
|
||||
|
return new SqlQuerySpec(query, parameters); |
||||
|
} |
||||
|
|
||||
|
public static SqlQuerySpec LastPosition(string streamName) |
||||
|
{ |
||||
|
var query = |
||||
|
$"SELECT TOP 1 " + |
||||
|
$" e.eventStreamOffset," + |
||||
|
$" e.eventsCount " + |
||||
|
$"FROM {Constants.Collection} e " + |
||||
|
$"WHERE " + |
||||
|
$" e.eventStream = @name " + |
||||
|
$"ORDER BY e.eventStreamOffset DESC"; |
||||
|
|
||||
|
var parameters = new SqlParameterCollection |
||||
|
{ |
||||
|
new SqlParameter("@name", streamName) |
||||
|
}; |
||||
|
|
||||
|
return new SqlQuerySpec(query, parameters); |
||||
|
} |
||||
|
|
||||
|
public static SqlQuerySpec ByStreamName(string streamName, long streamPosition = 0) |
||||
|
{ |
||||
|
var query = |
||||
|
$"SELECT * " + |
||||
|
$"FROM {Constants.Collection} e " + |
||||
|
$"WHERE " + |
||||
|
$" e.eventStream = @name " + |
||||
|
$"AND e.eventStreamOffset >= @position " + |
||||
|
$"ORDER BY e.eventStreamOffset ASC"; |
||||
|
|
||||
|
var parameters = new SqlParameterCollection |
||||
|
{ |
||||
|
new SqlParameter("@name", streamName), |
||||
|
new SqlParameter("@position", streamPosition) |
||||
|
}; |
||||
|
|
||||
|
return new SqlQuerySpec(query, parameters); |
||||
|
} |
||||
|
|
||||
|
public static SqlQuerySpec CreateByProperty(string property, object value, StreamPosition streamPosition) |
||||
|
{ |
||||
|
var filters = new List<string>(); |
||||
|
|
||||
|
var parameters = new SqlParameterCollection(); |
||||
|
|
||||
|
filters.ForPosition(parameters, streamPosition); |
||||
|
filters.ForProperty(parameters, property, value); |
||||
|
|
||||
|
return BuildQuery(filters, parameters); |
||||
|
} |
||||
|
|
||||
|
public static SqlQuerySpec CreateByFilter(string streamFilter, StreamPosition streamPosition) |
||||
|
{ |
||||
|
var filters = new List<string>(); |
||||
|
|
||||
|
var parameters = new SqlParameterCollection(); |
||||
|
|
||||
|
filters.ForPosition(parameters, streamPosition); |
||||
|
filters.ForRegex(parameters, streamFilter); |
||||
|
|
||||
|
return BuildQuery(filters, parameters); |
||||
|
} |
||||
|
|
||||
|
private static SqlQuerySpec BuildQuery(List<string> filters, SqlParameterCollection parameters) |
||||
|
{ |
||||
|
var query = $"SELECT * FROM {Constants.Collection} e WHERE {string.Join(" AND ", filters)} ORDER BY e.timestamp"; |
||||
|
|
||||
|
return new SqlQuerySpec(query, parameters); |
||||
|
} |
||||
|
|
||||
|
private static void ForProperty(this List<string> filters, SqlParameterCollection parameters, string property, object value) |
||||
|
{ |
||||
|
filters.Add($"ARRAY_CONTAINS(e.events, {{ \"header\": {{ \"{property}\": @value }} }}, true)"); |
||||
|
|
||||
|
parameters.Add(new SqlParameter("@value", value)); |
||||
|
} |
||||
|
|
||||
|
private static void ForRegex(this List<string> filters, SqlParameterCollection parameters, string streamFilter) |
||||
|
{ |
||||
|
if (!StreamFilter.IsAll(streamFilter)) |
||||
|
{ |
||||
|
if (streamFilter.Contains("^")) |
||||
|
{ |
||||
|
filters.Add($"STARTSWITH(e.eventStream, @filter)"); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
filters.Add($"e.eventStream = @filter"); |
||||
|
} |
||||
|
|
||||
|
parameters.Add(new SqlParameter("@filter", streamFilter)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static void ForPosition(this List<string> filters, SqlParameterCollection parameters, StreamPosition streamPosition) |
||||
|
{ |
||||
|
if (streamPosition.IsEndOfCommit) |
||||
|
{ |
||||
|
filters.Add($"e.timestamp > @time"); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
filters.Add($"e.timestamp >= @time"); |
||||
|
} |
||||
|
|
||||
|
parameters.Add(new SqlParameter("@time", streamPosition.Timestamp)); |
||||
|
} |
||||
|
|
||||
|
public static EventPredicate CreateExpression(string property, object value) |
||||
|
{ |
||||
|
if (!string.IsNullOrWhiteSpace(property)) |
||||
|
{ |
||||
|
var jsonValue = JsonValue.Create(value); |
||||
|
|
||||
|
return x => x.Headers.TryGetValue(property, out var p) && p.Equals(jsonValue); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
return x => true; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,62 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Linq; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Azure.Documents; |
||||
|
using Microsoft.Azure.Documents.Client; |
||||
|
using Microsoft.Azure.Documents.Linq; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal static class FilterExtensions |
||||
|
{ |
||||
|
public static async Task<T> FirstOrDefaultAsync<T>(this IQueryable<T> queryable, CancellationToken ct = default) |
||||
|
{ |
||||
|
var documentQuery = queryable.AsDocumentQuery(); |
||||
|
|
||||
|
using (documentQuery) |
||||
|
{ |
||||
|
if (documentQuery.HasMoreResults) |
||||
|
{ |
||||
|
var results = await documentQuery.ExecuteNextAsync<T>(ct); |
||||
|
|
||||
|
return results.FirstOrDefault(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return default; |
||||
|
} |
||||
|
|
||||
|
public static Task QueryAsync(this DocumentClient documentClient, Uri collectionUri, SqlQuerySpec querySpec, Func<CosmosDbEventCommit, Task> handler, CancellationToken ct = default) |
||||
|
{ |
||||
|
var query = documentClient.CreateDocumentQuery<CosmosDbEventCommit>(collectionUri, querySpec); |
||||
|
|
||||
|
return query.QueryAsync(handler, ct); |
||||
|
} |
||||
|
|
||||
|
public static async Task QueryAsync<T>(this IQueryable<T> queryable, Func<T, Task> handler, CancellationToken ct = default) |
||||
|
{ |
||||
|
var documentQuery = queryable.AsDocumentQuery(); |
||||
|
|
||||
|
using (documentQuery) |
||||
|
{ |
||||
|
while (documentQuery.HasMoreResults && !ct.IsCancellationRequested) |
||||
|
{ |
||||
|
var items = await documentQuery.ExecuteNextAsync<T>(ct); |
||||
|
|
||||
|
foreach (var item in items) |
||||
|
{ |
||||
|
await handler(item); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,55 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
internal sealed class StreamPosition |
||||
|
{ |
||||
|
public long Timestamp { get; } |
||||
|
|
||||
|
public long CommitOffset { get; } |
||||
|
|
||||
|
public long CommitSize { get; } |
||||
|
|
||||
|
public bool IsEndOfCommit |
||||
|
{ |
||||
|
get { return CommitOffset == CommitSize - 1; } |
||||
|
} |
||||
|
|
||||
|
public StreamPosition(long timestamp, long commitOffset, long commitSize) |
||||
|
{ |
||||
|
Timestamp = timestamp; |
||||
|
|
||||
|
CommitOffset = commitOffset; |
||||
|
CommitSize = commitSize; |
||||
|
} |
||||
|
|
||||
|
public static implicit operator string(StreamPosition position) |
||||
|
{ |
||||
|
var parts = new object[] |
||||
|
{ |
||||
|
position.Timestamp, |
||||
|
position.CommitOffset, |
||||
|
position.CommitSize |
||||
|
}; |
||||
|
|
||||
|
return string.Join("-", parts); |
||||
|
} |
||||
|
|
||||
|
public static implicit operator StreamPosition(string position) |
||||
|
{ |
||||
|
if (!string.IsNullOrWhiteSpace(position)) |
||||
|
{ |
||||
|
var parts = position.Split('-'); |
||||
|
|
||||
|
return new StreamPosition(long.Parse(parts[0]), long.Parse(parts[1]), long.Parse(parts[2])); |
||||
|
} |
||||
|
|
||||
|
return new StreamPosition(0, -1, -1); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,14 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
namespace Squidex.Infrastructure.MongoDb |
||||
|
{ |
||||
|
public sealed class MongoDbOptions |
||||
|
{ |
||||
|
public bool IsCosmosDb { get; set; } |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,22 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
public static class StreamFilter |
||||
|
{ |
||||
|
public static bool IsAll(string filter) |
||||
|
{ |
||||
|
return string.IsNullOrWhiteSpace(filter) |
||||
|
|| string.Equals(filter, ".*", StringComparison.OrdinalIgnoreCase) |
||||
|
|| string.Equals(filter, "(.*)", StringComparison.OrdinalIgnoreCase) |
||||
|
|| string.Equals(filter, "(.*?)", StringComparison.OrdinalIgnoreCase); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,144 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Threading.Tasks; |
||||
|
using FakeItEasy; |
||||
|
using Orleans; |
||||
|
using Squidex.Domain.Apps.Core.Apps; |
||||
|
using Squidex.Domain.Apps.Core.Contents; |
||||
|
using Squidex.Domain.Apps.Entities.Apps; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.Log; |
||||
|
using Squidex.Infrastructure.Orleans; |
||||
|
using Xunit; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public class GrainTextIndexerTests |
||||
|
{ |
||||
|
private readonly IGrainFactory grainFactory = A.Fake<IGrainFactory>(); |
||||
|
private readonly ITextIndexerGrain grain = A.Fake<ITextIndexerGrain>(); |
||||
|
private readonly Guid schemaId = Guid.NewGuid(); |
||||
|
private readonly Guid contentId = Guid.NewGuid(); |
||||
|
private readonly GrainTextIndexer sut; |
||||
|
|
||||
|
public GrainTextIndexerTests() |
||||
|
{ |
||||
|
A.CallTo(() => grainFactory.GetGrain<ITextIndexerGrain>(schemaId, null)) |
||||
|
.Returns(grain); |
||||
|
|
||||
|
sut = new GrainTextIndexer(grainFactory, A.Fake<ISemanticLog>()); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_call_grain_when_deleting_entry() |
||||
|
{ |
||||
|
await sut.DeleteAsync(schemaId, contentId); |
||||
|
|
||||
|
A.CallTo(() => grain.DeleteAsync(contentId)) |
||||
|
.MustHaveHappened(); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_catch_exception_when_deleting_failed() |
||||
|
{ |
||||
|
A.CallTo(() => grain.DeleteAsync(contentId)) |
||||
|
.Throws(new InvalidOperationException()); |
||||
|
|
||||
|
await sut.DeleteAsync(schemaId, contentId); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_call_grain_when_indexing_data() |
||||
|
{ |
||||
|
var data = new NamedContentData(); |
||||
|
var dataDraft = new NamedContentData(); |
||||
|
|
||||
|
await sut.IndexAsync(schemaId, contentId, data, dataDraft); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft))) |
||||
|
.MustHaveHappened(); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft))) |
||||
|
.MustHaveHappened(); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_not_call_grain_when_data_is_null() |
||||
|
{ |
||||
|
var dataDraft = new NamedContentData(); |
||||
|
|
||||
|
await sut.IndexAsync(schemaId, contentId, null, dataDraft); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => !x.Value.IsDraft))) |
||||
|
.MustNotHaveHappened(); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft))) |
||||
|
.MustHaveHappened(); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_not_call_grain_when_data_draft_is_null() |
||||
|
{ |
||||
|
var data = new NamedContentData(); |
||||
|
|
||||
|
await sut.IndexAsync(schemaId, contentId, data, null); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft))) |
||||
|
.MustHaveHappened(); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.IsDraft))) |
||||
|
.MustNotHaveHappened(); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_catch_exception_when_indexing_failed() |
||||
|
{ |
||||
|
var data = new NamedContentData(); |
||||
|
|
||||
|
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.Ignored)) |
||||
|
.Throws(new InvalidOperationException()); |
||||
|
|
||||
|
await sut.IndexAsync(schemaId, contentId, data, null); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_call_grain_when_searching() |
||||
|
{ |
||||
|
var foundIds = new List<Guid> { Guid.NewGuid() }; |
||||
|
|
||||
|
A.CallTo(() => grain.SearchAsync("Search", A<SearchContext>.Ignored)) |
||||
|
.Returns(foundIds); |
||||
|
|
||||
|
var ids = await sut.SearchAsync("Search", GetApp(), schemaId, true); |
||||
|
|
||||
|
Assert.Equal(foundIds, ids); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_not_call_grain_when_input_is_empty() |
||||
|
{ |
||||
|
var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, false); |
||||
|
|
||||
|
Assert.Null(ids); |
||||
|
|
||||
|
A.CallTo(() => grain.SearchAsync(A<string>.Ignored, A<SearchContext>.Ignored)) |
||||
|
.MustNotHaveHappened(); |
||||
|
} |
||||
|
|
||||
|
private static IAppEntity GetApp() |
||||
|
{ |
||||
|
var app = A.Fake<IAppEntity>(); |
||||
|
|
||||
|
A.CallTo(() => app.LanguagesConfig).Returns(LanguagesConfig.Build(Language.EN, Language.DE)); |
||||
|
|
||||
|
return app; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,178 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Threading.Tasks; |
||||
|
using Squidex.Domain.Apps.Core.Contents; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.Assets; |
||||
|
using Xunit; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public class TextIndexerGrainTests : IDisposable |
||||
|
{ |
||||
|
private readonly Guid schemaId = Guid.NewGuid(); |
||||
|
private readonly List<Guid> ids1 = new List<Guid> { Guid.NewGuid() }; |
||||
|
private readonly List<Guid> ids2 = new List<Guid> { Guid.NewGuid() }; |
||||
|
private readonly SearchContext context; |
||||
|
private readonly IAssetStore assetStore = new MemoryAssetStore(); |
||||
|
private readonly TextIndexerGrain sut; |
||||
|
|
||||
|
public TextIndexerGrainTests() |
||||
|
{ |
||||
|
context = new SearchContext |
||||
|
{ |
||||
|
Languages = new HashSet<string> { "de", "en" } |
||||
|
}; |
||||
|
|
||||
|
sut = new TextIndexerGrain(assetStore); |
||||
|
sut.ActivateAsync(schemaId).Wait(); |
||||
|
} |
||||
|
|
||||
|
public void Dispose() |
||||
|
{ |
||||
|
sut.OnDeactivateAsync().Wait(); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_read_index_and_retrieve() |
||||
|
{ |
||||
|
await AddInvariantContent(); |
||||
|
|
||||
|
await sut.DeactivateAsync(); |
||||
|
|
||||
|
var other = new TextIndexerGrain(assetStore); |
||||
|
try |
||||
|
{ |
||||
|
await other.ActivateAsync(schemaId); |
||||
|
|
||||
|
var foundHello = await other.SearchAsync("Hello", context); |
||||
|
var foundWorld = await other.SearchAsync("World", context); |
||||
|
|
||||
|
Assert.Equal(ids1, foundHello); |
||||
|
Assert.Equal(ids2, foundWorld); |
||||
|
} |
||||
|
finally |
||||
|
{ |
||||
|
await other.OnDeactivateAsync(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_index_invariant_content_and_retrieve() |
||||
|
{ |
||||
|
await AddInvariantContent(); |
||||
|
|
||||
|
var foundHello = await sut.SearchAsync("Hello", context); |
||||
|
var foundWorld = await sut.SearchAsync("World", context); |
||||
|
|
||||
|
Assert.Equal(ids1, foundHello); |
||||
|
Assert.Equal(ids2, foundWorld); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() |
||||
|
{ |
||||
|
await AddInvariantContent(); |
||||
|
|
||||
|
var foundHello = await sut.SearchAsync("helo~", context); |
||||
|
var foundWorld = await sut.SearchAsync("wold~", context); |
||||
|
|
||||
|
Assert.Equal(ids1, foundHello); |
||||
|
Assert.Equal(ids2, foundWorld); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_delete_documents_from_index() |
||||
|
{ |
||||
|
await AddInvariantContent(); |
||||
|
|
||||
|
await sut.DeleteAsync(ids1[0]); |
||||
|
await sut.FlushAsync(); |
||||
|
|
||||
|
var helloIds = await sut.SearchAsync("Hello", context); |
||||
|
|
||||
|
var worldIds = await sut.SearchAsync("World", context); |
||||
|
|
||||
|
Assert.Empty(helloIds); |
||||
|
Assert.Equal(ids2, worldIds); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_index_localized_content_and_retrieve() |
||||
|
{ |
||||
|
await AddLocalizedContent(); |
||||
|
|
||||
|
var german1 = await sut.SearchAsync("Stadt", context); |
||||
|
var german2 = await sut.SearchAsync("and", context); |
||||
|
|
||||
|
var germanStopwordsIds = await sut.SearchAsync("und", context); |
||||
|
|
||||
|
Assert.Equal(ids1, german1); |
||||
|
Assert.Equal(ids1, german2); |
||||
|
Assert.Equal(ids2, germanStopwordsIds); |
||||
|
|
||||
|
var english1 = await sut.SearchAsync("City", context); |
||||
|
var english2 = await sut.SearchAsync("und", context); |
||||
|
|
||||
|
var englishStopwordsIds = await sut.SearchAsync("and", context); |
||||
|
|
||||
|
Assert.Equal(ids2, english1); |
||||
|
Assert.Equal(ids2, english2); |
||||
|
Assert.Equal(ids1, englishStopwordsIds); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_throw_exception_for_invalid_query() |
||||
|
{ |
||||
|
await AddInvariantContent(); |
||||
|
|
||||
|
await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context)); |
||||
|
} |
||||
|
|
||||
|
private async Task AddLocalizedContent() |
||||
|
{ |
||||
|
var germanData = |
||||
|
new NamedContentData() |
||||
|
.AddField("localized", |
||||
|
new ContentFieldData() |
||||
|
.AddValue("de", "Stadt und Umgebung and whatever")); |
||||
|
|
||||
|
var englishData = |
||||
|
new NamedContentData() |
||||
|
.AddField("localized", |
||||
|
new ContentFieldData() |
||||
|
.AddValue("en", "City and Surroundings und sonstiges")); |
||||
|
|
||||
|
await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }); |
||||
|
await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }); |
||||
|
await sut.FlushAsync(); |
||||
|
} |
||||
|
|
||||
|
private async Task AddInvariantContent() |
||||
|
{ |
||||
|
var data1 = |
||||
|
new NamedContentData() |
||||
|
.AddField("test", |
||||
|
new ContentFieldData() |
||||
|
.AddValue("iv", "Hello")); |
||||
|
|
||||
|
var data2 = |
||||
|
new NamedContentData() |
||||
|
.AddField("test", |
||||
|
new ContentFieldData() |
||||
|
.AddValue("iv", "World")); |
||||
|
|
||||
|
await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }); |
||||
|
await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }); |
||||
|
|
||||
|
await sut.FlushAsync(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,35 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using Microsoft.Azure.Documents.Client; |
||||
|
using Squidex.Infrastructure.TestHelpers; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
public sealed class CosmosDbEventStoreFixture : IDisposable |
||||
|
{ |
||||
|
private const string EmulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="; |
||||
|
private const string EmulatorUri = "https://localhost:8081"; |
||||
|
private readonly DocumentClient client; |
||||
|
|
||||
|
public CosmosDbEventStore EventStore { get; } |
||||
|
|
||||
|
public CosmosDbEventStoreFixture() |
||||
|
{ |
||||
|
client = new DocumentClient(new Uri(EmulatorUri), EmulatorKey, JsonHelper.DefaultSettings()); |
||||
|
|
||||
|
EventStore = new CosmosDbEventStore(client, EmulatorKey, "Test", JsonHelper.DefaultSettings()); |
||||
|
EventStore.InitializeAsync().Wait(); |
||||
|
} |
||||
|
|
||||
|
public void Dispose() |
||||
|
{ |
||||
|
client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri("Test")).Wait(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,29 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using Xunit; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing |
||||
|
{ |
||||
|
[Trait("Category", "Dependencies")] |
||||
|
public class CosmosDbEventStoreTests : EventStoreTests<CosmosDbEventStore>, IClassFixture<CosmosDbEventStoreFixture> |
||||
|
{ |
||||
|
private readonly CosmosDbEventStoreFixture fixture; |
||||
|
|
||||
|
protected override int SubscriptionDelayInMs { get; } = 1000; |
||||
|
|
||||
|
public CosmosDbEventStoreTests(CosmosDbEventStoreFixture fixture) |
||||
|
{ |
||||
|
this.fixture = fixture; |
||||
|
} |
||||
|
|
||||
|
public override CosmosDbEventStore CreateStore() |
||||
|
{ |
||||
|
return fixture.EventStore; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue