mirror of https://github.com/Squidex/squidex.git
Browse Source
* Sharding V1 * Revert naming. * Revert more changes. * Fix text index. * Full text improvement. * Improved full text. * Update names. * Revert search.pull/1056/head
Sebastian Stehle
2 years ago
committed by
GitHub
GitHub
51 changed files with 1651 additions and 498 deletions
@ -0,0 +1,90 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using MongoDB.Driver; |
|||
using Squidex.Domain.Apps.Core.Assets; |
|||
using Squidex.Domain.Apps.Entities.Assets.Repositories; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Assets; |
|||
|
|||
public sealed class MongoShardedAssetRepository : ShardedSnapshotStore<MongoAssetRepository, Asset>, IAssetRepository |
|||
{ |
|||
public MongoShardedAssetRepository(IShardingStrategy sharding, Func<string, MongoAssetRepository> factory) |
|||
: base(sharding, factory) |
|||
{ |
|||
} |
|||
|
|||
public IEnumerable<IMongoCollection<MongoAssetEntity>> GetInternalCollections() |
|||
{ |
|||
return Shards.Select(x => x.GetInternalCollection()); |
|||
} |
|||
|
|||
public async Task<Asset?> FindAssetAsync(DomainId id, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Asset? result = null; |
|||
|
|||
foreach (var shard in Shards) |
|||
{ |
|||
if ((result = await shard.FindAssetAsync(id, ct)) != null) |
|||
{ |
|||
return result; |
|||
} |
|||
} |
|||
|
|||
return result; |
|||
} |
|||
|
|||
public Task<Asset?> FindAssetAsync(DomainId appId, DomainId id, bool allowDeleted, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).FindAssetAsync(appId, id, allowDeleted, ct); |
|||
} |
|||
|
|||
public Task<Asset?> FindAssetByHashAsync(DomainId appId, string hash, string fileName, long fileSize, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).FindAssetByHashAsync(appId, hash, fileName, fileSize, ct); |
|||
} |
|||
|
|||
public Task<Asset?> FindAssetBySlugAsync(DomainId appId, string slug, bool allowDeleted, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).FindAssetBySlugAsync(appId, slug, allowDeleted, ct); |
|||
} |
|||
|
|||
public Task<IResultList<Asset>> QueryAsync(DomainId appId, DomainId? parentId, Q q, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).QueryAsync(appId, parentId, q, ct); |
|||
} |
|||
|
|||
public Task<IReadOnlyList<DomainId>> QueryChildIdsAsync(DomainId appId, DomainId parentId, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).QueryChildIdsAsync(appId, parentId, ct); |
|||
} |
|||
|
|||
public Task<IReadOnlyList<DomainId>> QueryIdsAsync(DomainId appId, HashSet<DomainId> ids, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).QueryIdsAsync(appId, ids, ct); |
|||
} |
|||
|
|||
public IAsyncEnumerable<Asset> StreamAll(DomainId appId, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).StreamAll(appId, ct); |
|||
} |
|||
|
|||
protected override string GetShardKey(Asset state) |
|||
{ |
|||
return GetShardKey(state.AppId.Id); |
|||
} |
|||
} |
|||
@ -0,0 +1,183 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Text; |
|||
using MongoDB.Bson.Serialization; |
|||
using MongoDB.Bson.Serialization.Serializers; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb; |
|||
|
|||
public sealed class BsonUniqueContentIdSerializer : SerializerBase<UniqueContentId> |
|||
{ |
|||
private const byte GuidLength = 16; |
|||
private static readonly BsonUniqueContentIdSerializer Instance = new BsonUniqueContentIdSerializer(); |
|||
|
|||
public static void Register() |
|||
{ |
|||
BsonSerializer.TryRegisterSerializer(Instance); |
|||
} |
|||
|
|||
private BsonUniqueContentIdSerializer() |
|||
{ |
|||
} |
|||
|
|||
public static UniqueContentId NextAppId(DomainId appId) |
|||
{ |
|||
static void IncrementByteArray(byte[] bytes) |
|||
{ |
|||
for (var i = 0; i < bytes.Length; i++) |
|||
{ |
|||
var value = bytes[i]; |
|||
if (value < byte.MaxValue) |
|||
{ |
|||
value += 1; |
|||
bytes[i] = value; |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
if (Guid.TryParse(appId.ToString(), out var id)) |
|||
{ |
|||
var bytes = id.ToByteArray(); |
|||
|
|||
IncrementByteArray(bytes); |
|||
|
|||
return new UniqueContentId(DomainId.Create(new Guid(bytes)), DomainId.Empty); |
|||
} |
|||
else |
|||
{ |
|||
var bytes = Encoding.UTF8.GetBytes(appId.ToString()); |
|||
|
|||
IncrementByteArray(bytes); |
|||
|
|||
return new UniqueContentId(DomainId.Create(Encoding.UTF8.GetString(bytes)), DomainId.Empty); |
|||
} |
|||
} |
|||
|
|||
public override UniqueContentId Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) |
|||
{ |
|||
var buffer = context.Reader.ReadBytes()!; |
|||
var offset = 0; |
|||
|
|||
static DomainId ReadId(byte[] buffer, ref int offset) |
|||
{ |
|||
DomainId id; |
|||
|
|||
// If we have reached the end of the buffer then
|
|||
if (offset >= buffer.Length) |
|||
{ |
|||
return default; |
|||
} |
|||
|
|||
var length = buffer[offset++]; |
|||
// Special length indicator for all guids.
|
|||
if (length == 0xFF) |
|||
{ |
|||
id = DomainId.Create(new Guid(buffer.AsSpan(offset, GuidLength))); |
|||
offset += GuidLength; |
|||
} |
|||
else |
|||
{ |
|||
id = DomainId.Create(Encoding.UTF8.GetString(buffer.AsSpan(offset, length))); |
|||
offset += length; |
|||
} |
|||
|
|||
return id; |
|||
} |
|||
|
|||
return new UniqueContentId(ReadId(buffer, ref offset), ReadId(buffer, ref offset)); |
|||
} |
|||
|
|||
public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, UniqueContentId value) |
|||
{ |
|||
var appId = CheckId(value.AppId); |
|||
|
|||
var contentId = CheckId(value.ContentId); |
|||
|
|||
var isEmptyContentId = |
|||
contentId.IsGuid && |
|||
contentId.Guid == default; |
|||
|
|||
// Do not write empty Ids to the buffer to allow prefix searches.
|
|||
var contentLength = !isEmptyContentId ? contentId.Length + 1 : 0; |
|||
|
|||
var bufferLength = appId.Length + 1 + contentLength; |
|||
var bufferArray = new byte[bufferLength]; |
|||
|
|||
var offset = Write(bufferArray, 0, |
|||
appId.IsGuid, |
|||
appId.Guid, |
|||
appId.Source, |
|||
appId.Length); |
|||
|
|||
if (!isEmptyContentId) |
|||
{ |
|||
// Do not write the empty content id, so we can search for app as well.
|
|||
Write(bufferArray, offset, |
|||
contentId.IsGuid, |
|||
contentId.Guid, |
|||
contentId.Source, |
|||
contentId.Length); |
|||
} |
|||
|
|||
static int Write(byte[] buffer, int offset, bool isGuid, Guid guid, string id, int idLength) |
|||
{ |
|||
if (isGuid) |
|||
{ |
|||
// Special length indicator for all guids.
|
|||
buffer[offset++] = 0xFF; |
|||
WriteGuid(buffer.AsSpan(offset), guid); |
|||
|
|||
return offset + GuidLength; |
|||
} |
|||
else |
|||
{ |
|||
// We assume that we use relatively small IDs, not longer than 254 bytes.
|
|||
buffer[offset++] = (byte)idLength; |
|||
WriteString(buffer.AsSpan(offset), id); |
|||
|
|||
return offset + idLength; |
|||
} |
|||
} |
|||
|
|||
context.Writer.WriteBytes(bufferArray); |
|||
} |
|||
|
|||
private static (int Length, bool IsGuid, Guid Guid, string Source) CheckId(DomainId id) |
|||
{ |
|||
var source = id.ToString(); |
|||
|
|||
var idIsGuid = Guid.TryParse(source, out var idGuid); |
|||
var idLength = GuidLength; |
|||
|
|||
if (!idIsGuid) |
|||
{ |
|||
idLength = (byte)Encoding.UTF8.GetByteCount(source); |
|||
|
|||
// We only use a single byte to write the length, therefore we do not allow large strings.
|
|||
if (idLength > 254) |
|||
{ |
|||
ThrowHelper.InvalidOperationException("Cannot write long IDs."); |
|||
} |
|||
} |
|||
|
|||
return (idLength, idIsGuid, idGuid, source); |
|||
} |
|||
|
|||
private static void WriteString(Span<byte> span, string id) |
|||
{ |
|||
Encoding.UTF8.GetBytes(id, span); |
|||
} |
|||
|
|||
private static void WriteGuid(Span<byte> span, Guid guid) |
|||
{ |
|||
guid.TryWriteBytes(span); |
|||
} |
|||
} |
|||
@ -0,0 +1,98 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Runtime.CompilerServices; |
|||
using NodaTime; |
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Domain.Apps.Core.Schemas; |
|||
using Squidex.Domain.Apps.Entities.Contents; |
|||
using Squidex.Domain.Apps.Entities.Contents.Repositories; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Queries; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Contents; |
|||
|
|||
public sealed class MongoShardedContentRepository : ShardedSnapshotStore<MongoContentRepository, WriteContent>, IContentRepository |
|||
{ |
|||
public MongoShardedContentRepository(IShardingStrategy sharding, Func<string, MongoContentRepository> factory) |
|||
: base(sharding, factory) |
|||
{ |
|||
} |
|||
|
|||
public Task<Content?> FindContentAsync(App app, Schema schema, DomainId id, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).FindContentAsync(app, schema, id, scope, ct); |
|||
} |
|||
|
|||
public Task<bool> HasReferrersAsync(App app, DomainId reference, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).HasReferrersAsync(app, reference, scope, ct); |
|||
} |
|||
|
|||
public Task<IResultList<Content>> QueryAsync(App app, List<Schema> schemas, Q q, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).QueryAsync(app, schemas, q, scope, ct); |
|||
} |
|||
|
|||
public Task<IResultList<Content>> QueryAsync(App app, Schema schema, Q q, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).QueryAsync(app, schema, q, scope, ct); |
|||
} |
|||
|
|||
public Task<IReadOnlyList<ContentIdStatus>> QueryIdsAsync(App app, Schema schema, FilterNode<ClrValue> filterNode, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).QueryIdsAsync(app, schema, filterNode, scope, ct); |
|||
} |
|||
|
|||
public Task<IReadOnlyList<ContentIdStatus>> QueryIdsAsync(App app, HashSet<DomainId> ids, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).QueryIdsAsync(app, ids, scope, ct); |
|||
} |
|||
|
|||
public Task ResetScheduledAsync(DomainId appId, DomainId id, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).ResetScheduledAsync(appId, id, scope, ct); |
|||
} |
|||
|
|||
public IAsyncEnumerable<Content> StreamAll(DomainId appId, HashSet<DomainId>? schemaIds, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).StreamAll(appId, schemaIds, scope, ct); |
|||
} |
|||
|
|||
public IAsyncEnumerable<Content> StreamReferencing(DomainId appId, DomainId references, int take, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(appId).StreamReferencing(appId, references, take, scope, ct); |
|||
} |
|||
|
|||
public async IAsyncEnumerable<Content> StreamScheduledWithoutDataAsync(Instant now, SearchScope scope, |
|||
[EnumeratorCancellation] CancellationToken ct = default) |
|||
{ |
|||
foreach (var shard in Shards) |
|||
{ |
|||
await foreach (var content in shard.StreamScheduledWithoutDataAsync(now, scope, ct)) |
|||
{ |
|||
yield return content; |
|||
} |
|||
} |
|||
} |
|||
|
|||
protected override string GetShardKey(WriteContent state) |
|||
{ |
|||
return GetShardKey(state.AppId.Id); |
|||
} |
|||
} |
|||
@ -0,0 +1,101 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Runtime.CompilerServices; |
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb; |
|||
|
|||
public abstract class ShardedSnapshotStore<T, TState> : ShardedService<T>, ISnapshotStore<TState>, IDeleter where T : ISnapshotStore<TState>, IDeleter |
|||
{ |
|||
protected ShardedSnapshotStore(IShardingStrategy sharding, Func<string, T> factory) |
|||
: base(sharding, factory) |
|||
{ |
|||
} |
|||
|
|||
protected abstract string GetShardKey(TState state); |
|||
|
|||
public Task WriteAsync(SnapshotWriteJob<TState> job, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var shard = Shard(GetShardKey(job.Value)); |
|||
|
|||
return shard.WriteAsync(job, ct); |
|||
} |
|||
|
|||
public Task<SnapshotResult<TState>> ReadAsync(DomainId key, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var shard = Shard(GetAppId(key)); |
|||
|
|||
return shard.ReadAsync(key, ct); |
|||
} |
|||
|
|||
public Task RemoveAsync(DomainId key, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var shard = Shard(GetAppId(key)); |
|||
|
|||
return shard.RemoveAsync(key, ct); |
|||
} |
|||
|
|||
public Task DeleteAppAsync(App app, |
|||
CancellationToken ct) |
|||
{ |
|||
var shard = Shard(app.Id); |
|||
|
|||
return shard.DeleteAppAsync(app, ct); |
|||
} |
|||
|
|||
public async IAsyncEnumerable<SnapshotResult<TState>> ReadAllAsync( |
|||
[EnumeratorCancellation] CancellationToken ct = default) |
|||
{ |
|||
foreach (var shard in Shards) |
|||
{ |
|||
await foreach (var asset in shard.ReadAllAsync(ct)) |
|||
{ |
|||
yield return asset; |
|||
} |
|||
} |
|||
} |
|||
|
|||
public async Task ClearAsync( |
|||
CancellationToken ct = default) |
|||
{ |
|||
foreach (var shard in Shards) |
|||
{ |
|||
await shard.ClearAsync(ct); |
|||
} |
|||
} |
|||
|
|||
public async Task WriteManyAsync(IEnumerable<SnapshotWriteJob<TState>> jobs, |
|||
CancellationToken ct = default) |
|||
{ |
|||
// Some commands might share a shared, therefore we don't group by app id.
|
|||
foreach (var byShard in jobs.GroupBy(c => GetShardKey(c.Value))) |
|||
{ |
|||
var shard = Shard(byShard.Key); |
|||
|
|||
await shard.WriteManyAsync(byShard.ToArray(), ct); |
|||
} |
|||
} |
|||
|
|||
private static DomainId GetAppId(DomainId key) |
|||
{ |
|||
// This is a leaky abstraction, but the only option to implement that in a fast way.
|
|||
var parts = key.ToString().Split(DomainId.IdSeparator); |
|||
|
|||
if (parts.Length != 2) |
|||
{ |
|||
throw new InvalidOperationException("The key does not contain an app id."); |
|||
} |
|||
|
|||
return DomainId.Create(parts[0]); |
|||
} |
|||
} |
|||
@ -0,0 +1,55 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Domain.Apps.Entities.Contents; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Text; |
|||
|
|||
public sealed class MongoShardedTextIndex<T> : ShardedService<MongoTextIndexBase<T>>, ITextIndex where T : class |
|||
{ |
|||
public MongoShardedTextIndex(IShardingStrategy sharding, Func<string, MongoTextIndexBase<T>> factory) |
|||
: base(sharding, factory) |
|||
{ |
|||
} |
|||
|
|||
public async Task ClearAsync( |
|||
CancellationToken ct = default) |
|||
{ |
|||
foreach (var shard in Shards) |
|||
{ |
|||
await shard.ClearAsync(ct); |
|||
} |
|||
} |
|||
|
|||
public async Task ExecuteAsync(IndexCommand[] commands, |
|||
CancellationToken ct = default) |
|||
{ |
|||
// Some commands might share a shared, therefore we don't group by app id.
|
|||
foreach (var byShard in commands.GroupBy(c => GetShardKey(c.UniqueContentId.AppId))) |
|||
{ |
|||
var shard = Shard(byShard.Key); |
|||
|
|||
await shard.ExecuteAsync(byShard.ToArray(), ct); |
|||
} |
|||
} |
|||
|
|||
public Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).SearchAsync(app, query, scope, ct); |
|||
} |
|||
|
|||
public Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Shard(app.Id).SearchAsync(app, query, scope, ct); |
|||
} |
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.State; |
|||
|
|||
public enum TextState |
|||
{ |
|||
Stage0_Draft__Stage1_None, |
|||
Stage0_Published__Stage1_None, |
|||
Stage0_Published__Stage1_Draft, |
|||
Stage1_Draft__Stage0_None, |
|||
Stage1_Published__Stage0_None, |
|||
Stage1_Published__Stage0_Draft, |
|||
Deleted, |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Infrastructure; |
|||
|
|||
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public record struct UniqueContentId(DomainId AppId, DomainId ContentId) |
|||
{ |
|||
} |
|||
@ -0,0 +1,68 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Hosting; |
|||
|
|||
namespace Squidex.Infrastructure.States; |
|||
|
|||
public abstract class ShardedService<T> : IInitializable |
|||
{ |
|||
private readonly Dictionary<string, T> shards = new Dictionary<string, T>(); |
|||
private readonly IShardingStrategy sharding; |
|||
private readonly Func<string, T> factory; |
|||
|
|||
protected IEnumerable<T> Shards => shards.Values; |
|||
|
|||
protected ShardedService(IShardingStrategy sharding, Func<string, T> factory) |
|||
{ |
|||
this.sharding = sharding; |
|||
this.factory = factory; |
|||
} |
|||
|
|||
public async Task InitializeAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
foreach (var shardKey in sharding.GetShardKeys()) |
|||
{ |
|||
var inner = factory(shardKey); |
|||
|
|||
if (inner is IInitializable initializable) |
|||
{ |
|||
await initializable.InitializeAsync(ct); |
|||
} |
|||
|
|||
shards[shardKey] = inner; |
|||
} |
|||
} |
|||
|
|||
public async Task ReleaseAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
foreach (var inner in shards.Values) |
|||
{ |
|||
if (inner is IInitializable initializable) |
|||
{ |
|||
await initializable.ReleaseAsync(ct); |
|||
} |
|||
} |
|||
} |
|||
|
|||
protected string GetShardKey<TKey>(TKey key) where TKey : notnull |
|||
{ |
|||
return sharding.GetShardKey(key); |
|||
} |
|||
|
|||
protected T Shard<TKey>(TKey key) where TKey : notnull |
|||
{ |
|||
return shards[GetShardKey(key)]; |
|||
} |
|||
|
|||
protected string GetShardKey(DomainId appId) |
|||
{ |
|||
return sharding.GetShardKey(appId); |
|||
} |
|||
} |
|||
@ -0,0 +1,66 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
#pragma warning disable MA0048 // File name must match type name
|
|||
|
|||
namespace Squidex.Infrastructure.States; |
|||
|
|||
public interface IShardingStrategy |
|||
{ |
|||
string GetShardKey<T>(T key) where T : notnull; |
|||
|
|||
IEnumerable<string> GetShardKeys(); |
|||
} |
|||
|
|||
public sealed class SingleSharding : IShardingStrategy |
|||
{ |
|||
public static readonly IShardingStrategy Instance = new SingleSharding(); |
|||
|
|||
private SingleSharding() |
|||
{ |
|||
} |
|||
|
|||
public string GetShardKey<T>(T key) where T : notnull |
|||
{ |
|||
return string.Empty; |
|||
} |
|||
|
|||
public IEnumerable<string> GetShardKeys() |
|||
{ |
|||
yield return string.Empty; |
|||
} |
|||
} |
|||
|
|||
public sealed class PartitionedSharding : IShardingStrategy |
|||
{ |
|||
private readonly int numPartitions; |
|||
|
|||
public PartitionedSharding(int numPartitions) |
|||
{ |
|||
this.numPartitions = numPartitions; |
|||
} |
|||
|
|||
public string GetShardKey<T>(T key) where T : notnull |
|||
{ |
|||
var partition = Math.Abs(key.GetHashCode()) % numPartitions; |
|||
|
|||
return GetShardKey(partition); |
|||
} |
|||
|
|||
public IEnumerable<string> GetShardKeys() |
|||
{ |
|||
for (var i = 0; i < numPartitions; i++) |
|||
{ |
|||
yield return GetShardKey(i); |
|||
} |
|||
} |
|||
|
|||
private static string GetShardKey(int partition) |
|||
{ |
|||
return $"_{partition}"; |
|||
} |
|||
} |
|||
@ -0,0 +1,90 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Core.TestHelpers; |
|||
using Squidex.Domain.Apps.Entities.MongoDb; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public class BsonUniqueContentIdSerializerTests |
|||
{ |
|||
public BsonUniqueContentIdSerializerTests() |
|||
{ |
|||
BsonUniqueContentIdSerializer.Register(); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_serialize_and_deserialize_guid_guid() |
|||
{ |
|||
var source = new UniqueContentId(DomainId.NewGuid(), DomainId.NewGuid()); |
|||
|
|||
var deserialized = source.SerializeAndDeserializeBson(); |
|||
|
|||
Assert.Equal(source, deserialized); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_serialize_and_deserialize_guid_custom() |
|||
{ |
|||
var source = new UniqueContentId(DomainId.NewGuid(), DomainId.Create("id42")); |
|||
|
|||
var deserialized = source.SerializeAndDeserializeBson(); |
|||
|
|||
Assert.Equal(source, deserialized); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_serialize_and_deserialize_guid_empty() |
|||
{ |
|||
var source = new UniqueContentId(DomainId.NewGuid(), DomainId.Empty); |
|||
|
|||
var deserialized = source.SerializeAndDeserializeBson(); |
|||
|
|||
Assert.Equal(source, deserialized); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_serialize_and_deserialize_custom_custom() |
|||
{ |
|||
var source = new UniqueContentId(DomainId.Create("id41"), DomainId.Create("id42")); |
|||
|
|||
var deserialized = source.SerializeAndDeserializeBson(); |
|||
|
|||
Assert.Equal(source, deserialized); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_serialize_and_deserialize_custom_guid() |
|||
{ |
|||
var source = new UniqueContentId(DomainId.Create("id42"), DomainId.NewGuid()); |
|||
|
|||
var deserialized = source.SerializeAndDeserializeBson(); |
|||
|
|||
Assert.Equal(source, deserialized); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_calculate_next_custom_id() |
|||
{ |
|||
var appId = DomainId.Create("x"); |
|||
|
|||
var actual = BsonUniqueContentIdSerializer.NextAppId(appId); |
|||
|
|||
Assert.Equal(new UniqueContentId(DomainId.Create("y"), DomainId.Empty), actual); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_calculate_next_guid_id() |
|||
{ |
|||
var appId = DomainId.Create("70fb3772-2ab5-4854-b421-054d2479a0f7"); |
|||
|
|||
var actual = BsonUniqueContentIdSerializer.NextAppId(appId); |
|||
|
|||
Assert.Equal(new UniqueContentId(DomainId.Create("70fb3773-2ab5-4854-b421-054d2479a0f7"), DomainId.Empty), actual); |
|||
} |
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Core.TestHelpers; |
|||
using Squidex.Domain.Apps.Entities.MongoDb.Text; |
|||
using Squidex.Domain.Apps.Entities.TestHelpers; |
|||
using Squidex.Infrastructure.MongoDb; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public sealed class MongoTextIndexerStateFixture : IAsyncLifetime |
|||
{ |
|||
public MongoTextIndexerState State { get; } |
|||
|
|||
public MongoTextIndexerStateFixture() |
|||
{ |
|||
TestUtils.SetupBson(); |
|||
|
|||
var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]!); |
|||
var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]!); |
|||
|
|||
State = new MongoTextIndexerState(mongoDatabase); |
|||
} |
|||
|
|||
public Task InitializeAsync() |
|||
{ |
|||
return State.InitializeAsync(default); |
|||
} |
|||
|
|||
public Task DisposeAsync() |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
} |
|||
@ -0,0 +1,95 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
#pragma warning disable SA1300 // Element should begin with upper-case letter
|
|||
|
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.State; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
[Trait("Category", "Dependencies")] |
|||
public class MongoTextIndexerStateTests : IClassFixture<MongoTextIndexerStateFixture> |
|||
{ |
|||
public MongoTextIndexerStateFixture _ { get; set; } |
|||
|
|||
public MongoTextIndexerStateTests(MongoTextIndexerStateFixture fixture) |
|||
{ |
|||
_ = fixture; |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_add_state() |
|||
{ |
|||
var appId = DomainId.NewGuid(); |
|||
var id1 = new UniqueContentId(appId, DomainId.NewGuid()); |
|||
var id2 = new UniqueContentId(appId, DomainId.NewGuid()); |
|||
var id3 = new UniqueContentId(appId, DomainId.NewGuid()); |
|||
|
|||
await _.State.SetAsync( |
|||
[ |
|||
new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None }, |
|||
new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft }, |
|||
new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None } |
|||
]); |
|||
|
|||
var actual = await _.State.GetAsync(HashSet.Of(id1, id2)); |
|||
|
|||
actual.Should().BeEquivalentTo(new Dictionary<UniqueContentId, TextContentState> |
|||
{ |
|||
[id1] = new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None }, |
|||
[id2] = new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft } |
|||
}); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_remove_state() |
|||
{ |
|||
var id = new UniqueContentId(DomainId.NewGuid(), DomainId.NewGuid()); |
|||
|
|||
await _.State.SetAsync( |
|||
[ |
|||
new TextContentState { UniqueContentId = id, State = TextState.Stage0_Draft__Stage1_None } |
|||
]); |
|||
|
|||
await _.State.SetAsync( |
|||
[ |
|||
new TextContentState { UniqueContentId = id, State = TextState.Deleted } |
|||
]); |
|||
|
|||
var actual = await _.State.GetAsync(HashSet.Of(id)); |
|||
|
|||
Assert.Empty(actual); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_remove_by_app_state() |
|||
{ |
|||
var appId1 = DomainId.NewGuid(); |
|||
var appId2 = DomainId.NewGuid(); |
|||
var id1 = new UniqueContentId(appId1, DomainId.NewGuid()); |
|||
var id2 = new UniqueContentId(appId1, DomainId.NewGuid()); |
|||
var id3 = new UniqueContentId(appId2, DomainId.NewGuid()); |
|||
|
|||
await _.State.SetAsync( |
|||
[ |
|||
new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None }, |
|||
new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft }, |
|||
new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None } |
|||
]); |
|||
|
|||
await ((IDeleter)_.State).DeleteAppAsync(new App { Id = appId1 }, default); |
|||
|
|||
var actual = await _.State.GetAsync(HashSet.Of(id1, id2, id3)); |
|||
|
|||
actual.Should().BeEquivalentTo(new Dictionary<UniqueContentId, TextContentState> |
|||
{ |
|||
[id3] = new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None } |
|||
}); |
|||
} |
|||
} |
|||
@ -0,0 +1,82 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Hosting; |
|||
|
|||
namespace Squidex.Infrastructure.States; |
|||
|
|||
public class ShardedServiceTests |
|||
{ |
|||
private readonly IInner inner1 = A.Fake<IInner>(); |
|||
private readonly IInner inner2 = A.Fake<IInner>(); |
|||
private readonly TestSut sut; |
|||
|
|||
public interface IInner : IInitializable |
|||
{ |
|||
} |
|||
|
|||
private class TestSut : ShardedService<IInner> |
|||
{ |
|||
public TestSut(IShardingStrategy sharding, Func<string, IInner> factory) |
|||
: base(sharding, factory) |
|||
{ |
|||
} |
|||
|
|||
public IInner ExposeShard<TKey>(TKey key) where TKey : notnull |
|||
{ |
|||
return Shard(key); |
|||
} |
|||
} |
|||
|
|||
public ShardedServiceTests() |
|||
{ |
|||
sut = new TestSut(new PartitionedSharding(2), key => |
|||
{ |
|||
if (key == "_0") |
|||
{ |
|||
return inner1; |
|||
} |
|||
else |
|||
{ |
|||
return inner2; |
|||
} |
|||
}); |
|||
|
|||
sut.InitializeAsync(default).Wait(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_initialize_shards() |
|||
{ |
|||
await sut.InitializeAsync(default); |
|||
|
|||
A.CallTo(() => inner1.InitializeAsync(A<CancellationToken>._)) |
|||
.MustHaveHappened(); |
|||
|
|||
A.CallTo(() => inner2.InitializeAsync(A<CancellationToken>._)) |
|||
.MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_release_shards() |
|||
{ |
|||
await sut.ReleaseAsync(default); |
|||
|
|||
A.CallTo(() => inner1.ReleaseAsync(A<CancellationToken>._)) |
|||
.MustHaveHappened(); |
|||
|
|||
A.CallTo(() => inner2.ReleaseAsync(A<CancellationToken>._)) |
|||
.MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_provide_shards() |
|||
{ |
|||
Assert.Equal(inner1, sut.ExposeShard(0)); |
|||
Assert.Equal(inner2, sut.ExposeShard(1)); |
|||
} |
|||
} |
|||
@ -0,0 +1,57 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Infrastructure.States; |
|||
|
|||
public class ShardingTests |
|||
{ |
|||
[Fact] |
|||
public void Should_provide_single_shard_key() |
|||
{ |
|||
var strategy = SingleSharding.Instance; |
|||
|
|||
for (var i = 0; i < 1000; i++) |
|||
{ |
|||
var shardKey = strategy.GetShardKey(Guid.NewGuid()); |
|||
|
|||
Assert.Equal(string.Empty, shardKey); |
|||
} |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_provide_single_shard_keys() |
|||
{ |
|||
var strategy = SingleSharding.Instance; |
|||
|
|||
var shardKeys = strategy.GetShardKeys().ToArray(); |
|||
|
|||
Assert.Equal(new[] { string.Empty }, shardKeys); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_provide_partitioned_shard_key() |
|||
{ |
|||
var strategy = new PartitionedSharding(3); |
|||
|
|||
for (var i = 0; i < 1000; i++) |
|||
{ |
|||
var shardKey = strategy.GetShardKey(Guid.NewGuid()); |
|||
|
|||
Assert.True(shardKey is "_0" or "_1" or "_2"); |
|||
} |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_provide_partitioned_shard_keys() |
|||
{ |
|||
var strategy = new PartitionedSharding(3); |
|||
|
|||
var shardKeys = strategy.GetShardKeys().ToArray(); |
|||
|
|||
Assert.Equal(new[] { "_0", "_1", "_2" }, shardKeys); |
|||
} |
|||
} |
|||
Loading…
Reference in new issue