From 2c026df0e4b09f0bb1d1fa946c9c7f60f19134bb Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 30 Mar 2023 09:46:17 +0200 Subject: [PATCH] Improve eventstore queries (#982) * Update Oidc client. * Improve the queries and tests * Another performance improvement. * Fix tests * Unify code. * Fix readme. --- README.md | 2 +- .../Schemas/FieldCollection.cs | 2 +- .../HandleRules/RuleEventFormatter.cs | 2 +- .../Assets/Queries/AssetQueryParser.cs | 6 +- .../Assets/Queries/AssetQueryService.cs | 2 +- .../Backup/State/BackupState.cs | 2 +- .../Comments/CommentsResult.cs | 6 +- .../Contents/Queries/ContentQueryParser.cs | 6 +- .../MongoUserStore.cs | 4 +- .../EventSourcing/GetEventStore.cs | 4 +- .../EventSourcing/Utils.cs | 6 +- .../EventSourcing/FilterExtensions.cs | 20 +- .../EventSourcing/MongoEventStore.cs | 10 +- .../EventSourcing/MongoEventStore_Reader.cs | 62 +++--- .../EventSourcing/MongoEventStore_Writer.cs | 6 - .../EventSourcing/StreamPosition.cs | 23 +-- .../MongoDb/Batching.cs | 1 + .../MongoDb/BsonDefaultConventions.cs | 5 + .../MongoDb/MongoBase.cs | 1 - .../MongoDb/MongoClientFactory.cs | 5 - .../MongoDb/ProfilerCollection.cs | 43 +++++ .../MongoDb/ProfilerDocument.cs | 35 ++++ .../MongoDb/Queries/SortBuilder.cs | 2 +- .../EventSourcing/IEventNotifier.cs | 22 --- .../EventSourcing/IEventStore.cs | 4 +- .../Queries/PropertyPath.cs | 7 +- .../Queries/SortNode.cs | 18 +- .../States/Persistence.cs | 2 +- .../Config/Domain/EventSourcingServices.cs | 5 +- .../Squidex/Config/Domain/StoreServices.cs | 1 - .../TestHelpers/TestUtils.cs | 5 +- .../Assets/MongoDb/AssetsQueryFixture.cs | 57 +++--- .../MongoDb/AssetsQueryIntegrationTests.cs | 47 +++-- .../ContentsQueryDedicatedIntegrationTests.cs | 4 +- .../Contents/MongoDb/ContentsQueryFixture.cs | 69 +++---- .../MongoDb/ContentsQueryIntegrationTests.cs | 4 +- .../MongoDb/ContentsQueryTestsBase.cs | 25 +-- .../Contents/Text/AtlasTextIndexFixture.cs | 1 - .../Contents/Text/MongoTextIndexFixture.cs | 3 +- .../Schemas/MongoDb/SchemasHashFixture.cs | 3 +- .../DefaultUserResolverTests.cs | 4 +- .../EventConsumerProcessorIntegrationTests.cs | 5 +- ...onsumerProcessorIntegrationTests_Direct.cs | 4 +- .../EventSourcing/EventStoreTests.cs | 182 ++++++++++-------- .../EventSourcing/MongoEventStoreFixture.cs | 54 +++--- .../EventSourcing/MongoEventStoreTests.cs | 67 +++++++ .../MongoEventStoreTests_Direct.cs | 26 --- .../MongoEventStoreTests_ReplicaSet.cs | 26 --- .../EventSourcing/MongoParallelInsertTests.cs | 4 +- .../States/PersistenceEventSourcingTests.cs | 20 +- .../TestHelpers/TestUtils.cs | 5 +- .../appSettings.json | 2 +- .../Pipeline/AppResolverTests.cs | 6 +- .../Pipeline/TeamResolverTests.cs | 4 +- 54 files changed, 498 insertions(+), 443 deletions(-) create mode 100644 backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerCollection.cs create mode 100644 backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerDocument.cs delete mode 100644 backend/src/Squidex.Infrastructure/EventSourcing/IEventNotifier.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests.cs delete mode 100644 backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs delete mode 100644 backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs diff --git a/README.md b/README.md index 4630ca557..c9934bca4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![Squidex Logo](https://raw.githubusercontent.com/Squidex/squidex/master/media/logo-wide.png "Squidex") -# What is Squidex? +# What is Squidex?? Squidex is an open source headless CMS and content management hub. In contrast to a traditional CMS Squidex provides a rich API with OData filter and Swagger definitions. It is up to you to build your UI on top of it. It can be website, a native app or just another server. We build it with ASP.NET Core and CQRS and is tested for Windows and Linux on modern browsers. diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldCollection.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldCollection.cs index b4ed096cb..737c65b58 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldCollection.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldCollection.cs @@ -99,7 +99,7 @@ public sealed class FieldCollection where T : IField { Guard.NotNull(ids); - if (ids.Count != fieldsOrdered.Length || ids.Any(x => !ById.ContainsKey(x))) + if (ids.Count != fieldsOrdered.Length || ids.Exists(x => !ById.ContainsKey(x))) { ThrowHelper.ArgumentException("Ids must cover all fields.", nameof(ids)); } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs index 4734f3554..410b356e7 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs @@ -136,7 +136,7 @@ public class RuleEventFormatter var parts = BuildParts(text, @event); - if (parts.Any(x => !x.Var.IsCompleted)) + if (parts.Exists(x => !x.Var.IsCompleted)) { await ValueTaskEx.WhenAll(parts.Select(x => x.Var)); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs index b2c8cec7e..20e4bbf90 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs @@ -108,12 +108,12 @@ public class AssetQueryParser if (query.Sort.Count == 0) { - query.Sort.Add(new SortNode(new List { "lastModified" }, SortOrder.Descending)); + query.Sort.Add(new SortNode("lastModified", SortOrder.Descending)); } - if (!query.Sort.Any(x => string.Equals(x.Path.ToString(), "id", StringComparison.OrdinalIgnoreCase))) + if (!query.Sort.Exists(x => x.Path.Equals("id"))) { - query.Sort.Add(new SortNode(new List { "id" }, SortOrder.Ascending)); + query.Sort.Add(new SortNode("id", SortOrder.Ascending)); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryService.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryService.cs index 235d554ac..3e561f824 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryService.cs @@ -49,7 +49,7 @@ public sealed class AssetQueryService : IAssetQueryService { var folder = await FindFolderCoreAsync(appId, id, ct); - if (folder == null || result.Any(x => x.Id == folder.Id)) + if (folder == null || result.Exists(x => x.Id == folder.Id)) { result.Clear(); break; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs b/backend/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs index 22735552c..7f03d2804 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs @@ -16,7 +16,7 @@ public sealed class BackupState public void EnsureCanStart() { - if (Jobs.Any(x => x.Status == JobStatus.Started)) + if (Jobs.Exists(x => x.Status == JobStatus.Started)) { throw new DomainException(T.Get("backups.alreadyRunning")); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Comments/CommentsResult.cs b/backend/src/Squidex.Domain.Apps.Entities/Comments/CommentsResult.cs index 6b51dd13c..dd591e04d 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Comments/CommentsResult.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Comments/CommentsResult.cs @@ -34,11 +34,11 @@ public sealed class CommentsResult { var id = deleted.CommentId; - if (result.CreatedComments.Any(x => x.Id == id)) + if (result.CreatedComments.Exists(x => x.Id == id)) { result.CreatedComments.RemoveAll(x => x.Id == id); } - else if (result.UpdatedComments.Any(x => x.Id == id)) + else if (result.UpdatedComments.Exists(x => x.Id == id)) { result.UpdatedComments.RemoveAll(x => x.Id == id); result.DeletedComments.Add(id); @@ -75,7 +75,7 @@ public sealed class CommentsResult updated.Text, null); - if (result.CreatedComments.Any(x => x.Id == id)) + if (result.CreatedComments.Exists(x => x.Id == id)) { result.CreatedComments.RemoveAll(x => x.Id == id); result.CreatedComments.Add(comment); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs index caef13a8d..11c96f28b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs @@ -152,12 +152,12 @@ public class ContentQueryParser if (query.Sort.Count == 0) { - query.Sort.Add(new SortNode(new List { "lastModified" }, SortOrder.Descending)); + query.Sort.Add(new SortNode("lastModified", SortOrder.Descending)); } - if (!query.Sort.Any(x => string.Equals(x.Path.ToString(), "id", StringComparison.OrdinalIgnoreCase))) + if (!query.Sort.Exists(x => x.Path.Equals("id"))) { - query.Sort.Add(new SortNode(new List { "id" }, SortOrder.Ascending)); + query.Sort.Add(new SortNode("id", SortOrder.Ascending)); } } diff --git a/backend/src/Squidex.Domain.Users.MongoDb/MongoUserStore.cs b/backend/src/Squidex.Domain.Users.MongoDb/MongoUserStore.cs index efccb0c17..7950e824d 100644 --- a/backend/src/Squidex.Domain.Users.MongoDb/MongoUserStore.cs +++ b/backend/src/Squidex.Domain.Users.MongoDb/MongoUserStore.cs @@ -210,7 +210,7 @@ public sealed class MongoUserStore : public async Task FindByLoginAsync(string loginProvider, string providerKey, CancellationToken cancellationToken) { - var result = await Collection.Find(x => x.Logins.Any(y => y.LoginProvider == loginProvider && y.ProviderKey == providerKey)).FirstOrDefaultAsync(cancellationToken); + var result = await Collection.Find(x => x.Logins.Exists(y => y.LoginProvider == loginProvider && y.ProviderKey == providerKey)).FirstOrDefaultAsync(cancellationToken); return result; } @@ -218,7 +218,7 @@ public sealed class MongoUserStore : public async Task> GetUsersForClaimAsync(Claim claim, CancellationToken cancellationToken) { - var result = await Collection.Find(x => x.Claims.Any(y => y.Type == claim.Type && y.Value == claim.Value)).ToListAsync(cancellationToken); + var result = await Collection.Find(x => x.Claims.Exists(y => y.Type == claim.Type && y.Value == claim.Value)).ToListAsync(cancellationToken); return result.OfType().ToList(); } diff --git a/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs b/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs index 87bbbde3f..5e90f56a3 100644 --- a/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs +++ b/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs @@ -114,7 +114,7 @@ public sealed class GetEventStore : IEventStore, IInitializable } } - public async Task> QueryAsync(string streamName, long streamPosition = 0, + public async Task> QueryAsync(string streamName, long afterStreamPosition = EtagVersion.Empty, CancellationToken ct = default) { Guard.NotNullOrEmpty(streamName); @@ -123,7 +123,7 @@ public sealed class GetEventStore : IEventStore, IInitializable { var result = new List(); - var stream = QueryAsync(GetStreamName(streamName), streamPosition.ToPosition(), int.MaxValue, ct); + var stream = QueryAsync(GetStreamName(streamName), afterStreamPosition.ToPositionBefore(), int.MaxValue, ct); await foreach (var storedEvent in stream.IgnoreNotFound(ct)) { diff --git a/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Utils.cs b/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Utils.cs index 9723298a6..f2e233b28 100644 --- a/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Utils.cs +++ b/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Utils.cs @@ -18,14 +18,14 @@ public static class Utils return StreamRevision.FromInt64(version); } - public static StreamPosition ToPosition(this long version) + public static StreamPosition ToPositionBefore(this long version) { - if (version <= 0) + if (version < 0) { return StreamPosition.Start; } - return StreamPosition.FromInt64(version); + return StreamPosition.FromInt64(version - 1); } public static StreamPosition ToPosition(this string? position, bool inclusive) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/FilterExtensions.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/FilterExtensions.cs index 1a54f612f..3e9ec4f83 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/FilterExtensions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/FilterExtensions.cs @@ -11,6 +11,11 @@ namespace Squidex.Infrastructure.EventSourcing; internal static class FilterExtensions { + public static FilterDefinition ByOffset(long streamPosition) + { + return Builders.Filter.Gte(x => x.EventStreamOffset, streamPosition); + } + public static FilterDefinition ByPosition(StreamPosition streamPosition) { if (streamPosition.IsEndOfCommit) @@ -27,7 +32,7 @@ internal static class FilterExtensions { if (StreamFilter.IsAll(streamFilter)) { - return null; + return Builders.Filter.Exists(x => x.EventStream, true); } if (streamFilter.Contains('^', StringComparison.Ordinal)) @@ -57,7 +62,7 @@ internal static class FilterExtensions } } - public static IEnumerable Filtered(this MongoEventCommit commit, StreamPosition lastPosition) + public static IEnumerable Filtered(this MongoEventCommit commit, StreamPosition position) { var eventStreamOffset = commit.EventStreamOffset; @@ -68,7 +73,7 @@ internal static class FilterExtensions { eventStreamOffset++; - if (commitOffset > lastPosition.CommitOffset || commitTimestamp > lastPosition.Timestamp) + if (commitOffset > position.CommitOffset || commitTimestamp > position.Timestamp) { var eventData = @event.ToEventData(); var eventPosition = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); @@ -80,7 +85,12 @@ internal static class FilterExtensions } } - public static IEnumerable Filtered(this MongoEventCommit commit, long streamPosition = EtagVersion.Empty) + public static IEnumerable Filtered(this MongoEventCommit commit) + { + return commit.Filtered(EtagVersion.Empty); + } + + public static IEnumerable Filtered(this MongoEventCommit commit, long position) { var eventStreamOffset = commit.EventStreamOffset; @@ -91,7 +101,7 @@ internal static class FilterExtensions { eventStreamOffset++; - if (eventStreamOffset >= streamPosition) + if (eventStreamOffset > position) { var eventData = @event.ToEventData(); var eventPosition = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs index 4e4ba386b..c1f049f83 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs @@ -14,8 +14,6 @@ namespace Squidex.Infrastructure.EventSourcing; public partial class MongoEventStore : MongoRepositoryBase, IEventStore { - private readonly IEventNotifier notifier; - public IMongoCollection RawCollection { get => Database.GetCollection(CollectionName()); @@ -28,10 +26,9 @@ public partial class MongoEventStore : MongoRepositoryBase, IE public bool CanUseChangeStreams { get; private set; } - public MongoEventStore(IMongoDatabase database, IEventNotifier notifier) + public MongoEventStore(IMongoDatabase database) : base(database) { - this.notifier = notifier; } protected override string CollectionName() @@ -51,11 +48,8 @@ public partial class MongoEventStore : MongoRepositoryBase, IE { new CreateIndexModel( Index + .Ascending(x => x.EventStream) .Ascending(x => x.Timestamp)), - new CreateIndexModel( - Index - .Ascending(x => x.Timestamp) - .Ascending(x => x.EventStream)), new CreateIndexModel( Index .Descending(x => x.Timestamp) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs index 0e746f15a..54a2d64ec 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs @@ -9,7 +9,6 @@ using System.Runtime.CompilerServices; using MongoDB.Driver; using NodaTime; using Squidex.Infrastructure.MongoDb; -using EventFilter = MongoDB.Driver.FilterDefinition; #pragma warning disable MA0048 // File name must match type name @@ -59,7 +58,7 @@ public partial class MongoEventStore : MongoRepositoryBase, IE } } - public async Task> QueryAsync(string streamName, long streamPosition = 0, + public async Task> QueryAsync(string streamName, long afterStreamPosition = EtagVersion.Empty, CancellationToken ct = default) { Guard.NotNullOrEmpty(streamName); @@ -69,13 +68,27 @@ public partial class MongoEventStore : MongoRepositoryBase, IE var filter = Filter.And( Filter.Eq(x => x.EventStream, streamName), - Filter.Gte(x => x.EventStreamOffset, streamPosition - MaxCommitSize)); + Filter.Gte(x => x.EventStreamOffset, afterStreamPosition)); var commits = - await Collection.Find(filter).Sort(Sort.Ascending(x => x.Timestamp)) + await Collection.Find(filter) .ToListAsync(ct); - var result = commits.SelectMany(x => x.Filtered(streamPosition)).ToList(); + var result = Convert(commits, afterStreamPosition); + + if ((commits.Count == 0 || commits[0].EventStreamOffset != afterStreamPosition) && afterStreamPosition > EtagVersion.Empty) + { + filter = + Filter.And( + Filter.Eq(x => x.EventStream, streamName), + Filter.Lt(x => x.EventStreamOffset, afterStreamPosition)); + + commits = + await Collection.Find(filter).SortByDescending(x => x.EventStreamOffset).Limit(1) + .ToListAsync(ct); + + result = Convert(commits, afterStreamPosition).Concat(result).ToList(); + } return result; } @@ -88,21 +101,13 @@ public partial class MongoEventStore : MongoRepositoryBase, IE using (Telemetry.Activities.StartActivity("MongoEventStore/QueryManyAsync")) { - var position = EtagVersion.Empty; - - var filter = - Filter.And( - Filter.In(x => x.EventStream, streamNames), - Filter.Gte(x => x.EventStreamOffset, position)); + var filter = Filter.In(x => x.EventStream, streamNames); var commits = - await Collection.Find(filter).Sort(Sort.Ascending(x => x.Timestamp)) + await Collection.Find(filter) .ToListAsync(ct); - var result = commits.GroupBy(x => x.EventStream) - .ToDictionary( - x => x.Key, - x => (IReadOnlyList)x.SelectMany(y => y.Filtered(position)).ToList()); + var result = commits.GroupBy(x => x.EventStream).ToDictionary(x => x.Key, Convert); return result; } @@ -162,11 +167,11 @@ public partial class MongoEventStore : MongoRepositoryBase, IE var find = Collection.Find(filterDefinition) - .Limit(take).Sort(Sort.Ascending(x => x.Timestamp).Ascending(x => x.EventStream)); + .Limit(take); var taken = 0; - await foreach (var current in find.ToAsyncEnumerable(ct)) + await foreach (var current in find.ToAsyncEnumerable(ct).OrderBy(x => x.Timestamp).ThenBy(x => x.EventStream)) { foreach (var @event in current.Filtered(lastPosition)) { @@ -182,16 +187,25 @@ public partial class MongoEventStore : MongoRepositoryBase, IE } } - private static EventFilter CreateFilter(string? streamFilter, StreamPosition streamPosition) + private static IReadOnlyList Convert(IEnumerable commits) + { + return commits.OrderBy(x => x.EventStreamOffset).ThenBy(x => x.Timestamp).SelectMany(x => x.Filtered()).ToList(); + } + + private static IReadOnlyList Convert(IEnumerable commits, long streamPosition) + { + return commits.OrderBy(x => x.EventStreamOffset).ThenBy(x => x.Timestamp).SelectMany(x => x.Filtered(streamPosition)).ToList(); + } + + private static FilterDefinition CreateFilter(string? streamFilter, StreamPosition streamPosition) { - var byPosition = FilterExtensions.ByPosition(streamPosition); - var byStream = FilterExtensions.ByStream(streamFilter); + var filter = FilterExtensions.ByPosition(streamPosition); - if (byStream != null) + if (streamFilter != null) { - return Filter.And(byPosition, byStream); + return Filter.And(filter, FilterExtensions.ByStream(streamFilter)); } - return byPosition; + return filter; } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs index 12d6fe5fa..a2c09d7af 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs @@ -63,12 +63,6 @@ public partial class MongoEventStore try { await Collection.InsertOneAsync(commit, cancellationToken: ct); - - if (!CanUseChangeStreams) - { - notifier.NotifyEventsStored(streamName); - } - return; } catch (MongoWriteException ex) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/StreamPosition.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/StreamPosition.cs index 3b4c7e54d..230cb71f1 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/StreamPosition.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/StreamPosition.cs @@ -10,29 +10,16 @@ using MongoDB.Bson; using NodaTime; using Squidex.Infrastructure.ObjectPool; +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter +#pragma warning disable RECS0082 // Parameter has the same name as a member and hides it + namespace Squidex.Infrastructure.EventSourcing; -internal sealed class StreamPosition +internal sealed record StreamPosition(BsonTimestamp Timestamp, long CommitOffset, long CommitSize) { public static readonly StreamPosition Empty = new StreamPosition(new BsonTimestamp(0, 0), -1, -1); - public BsonTimestamp Timestamp { get; } - - public long CommitOffset { get; } - - public long CommitSize { get; } - - public bool IsEndOfCommit { get; } - - public StreamPosition(BsonTimestamp timestamp, long commitOffset, long commitSize) - { - Timestamp = timestamp; - - CommitOffset = commitOffset; - CommitSize = commitSize; - - IsEndOfCommit = CommitOffset == CommitSize - 1; - } + public bool IsEndOfCommit => CommitOffset == CommitSize - 1; public static implicit operator string(StreamPosition position) { diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Batching.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Batching.cs index cd98d8fa4..1be716fea 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Batching.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Batching.cs @@ -13,6 +13,7 @@ public static class Batching { public static readonly FindOptions Options = new FindOptions { + // Use a relatively small batch size to keep the memory pressure low. BatchSize = 200 }; } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDefaultConventions.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDefaultConventions.cs index 6e9591c5c..4ee7c52d2 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDefaultConventions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDefaultConventions.cs @@ -6,7 +6,9 @@ // ========================================================================== using MongoDB.Bson; +using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Conventions; +using MongoDB.Bson.Serialization.Serializers; namespace Squidex.Infrastructure.MongoDb; @@ -28,6 +30,9 @@ public static class BsonDefaultConventions new IgnoreExtraElementsConvention(true) }, t => true); + // Allow all types, independent from the actual assembly. + BsonSerializer.TryRegisterSerializer(new ObjectSerializer(type => true)); + isRegistered = true; } catch (BsonSerializationException) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs index 430eb377f..536cefff2 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs @@ -7,7 +7,6 @@ using MongoDB.Bson; using MongoDB.Driver; -using Squidex.Infrastructure.Json.Objects; #pragma warning disable RECS0108 // Warns about static fields in generic types diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoClientFactory.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoClientFactory.cs index fb078d590..c2fc50474 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoClientFactory.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoClientFactory.cs @@ -5,8 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using MongoDB.Bson.Serialization.Serializers; -using MongoDB.Bson.Serialization; using MongoDB.Driver; using MongoDB.Driver.Linq; using Squidex.Infrastructure.Json.Objects; @@ -17,9 +15,6 @@ public static class MongoClientFactory { public static MongoClient Create(string? connectionString, Action? configure = null) { - // Allow all types, independent from the actual assembly. - BsonSerializer.TryRegisterSerializer(new ObjectSerializer(type => true)); - BsonDefaultConventions.Register(); BsonDomainIdSerializer.Register(); BsonEscapedDictionarySerializer.Register(); diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerCollection.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerCollection.cs new file mode 100644 index 000000000..43730669f --- /dev/null +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerCollection.cs @@ -0,0 +1,43 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Bson; +using MongoDB.Driver; + +#pragma warning disable CA1822 // Mark members as static + +namespace Squidex.Infrastructure.MongoDb; + +public sealed class ProfilerCollection +{ + private readonly IMongoCollection collection; + + public string CollectionName => "system.profile"; + + public ProfilerCollection(IMongoDatabase database) + { + collection = database.GetCollection(CollectionName); + } + + public async Task> GetQueriesAsync(string collectionName, + CancellationToken ct = default) + { + var ns = $"{collection.Database.DatabaseNamespace.DatabaseName}.{collectionName}"; + + return await collection.Find(x => x.Operation == "query" && x.Namespace == ns).ToListAsync(ct); + } + + public async Task ClearAsync( + CancellationToken ct = default) + { + var database = collection.Database; + + await database.RunCommandAsync("{ profile : 0 }", cancellationToken: ct); + await database.DropCollectionAsync(ProfilerDocument.CollectionName, ct); + await database.RunCommandAsync("{ profile : 2 }", cancellationToken: ct); + } +} diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerDocument.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerDocument.cs new file mode 100644 index 000000000..2f8afb8f0 --- /dev/null +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/ProfilerDocument.cs @@ -0,0 +1,35 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace Squidex.Infrastructure.MongoDb; + +[BsonIgnoreExtraElements] +public sealed class ProfilerDocument +{ + public const string CollectionName = "system.profile"; + + [BsonElement("op")] + public string Operation { get; set; } + + [BsonElement("ns")] + public string Namespace { get; set; } + + [BsonElement("nreturned")] + public int NumDocuments { get; set; } + + [BsonElement("keysExamined")] + public int KeysExamined { get; set; } + + [BsonElement("docsExamined")] + public int DocsExamined { get; set; } + + [BsonElement("planSummary")] + public string PlanSummary { get; set; } +} diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/SortBuilder.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/SortBuilder.cs index a7ed38ef3..21e4e3ae9 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/SortBuilder.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/SortBuilder.cs @@ -14,7 +14,7 @@ public static class SortBuilder { public static SortDefinition? BuildSort(this ClrQuery query) { - if (query is { Sort: not null, Sort: { Count: > 0 } }) + if (query is { Sort: not null, Sort.Count: > 0 }) { var sorts = query.Sort.Select(OrderBy).ToList(); diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/IEventNotifier.cs b/backend/src/Squidex.Infrastructure/EventSourcing/IEventNotifier.cs deleted file mode 100644 index 830335931..000000000 --- a/backend/src/Squidex.Infrastructure/EventSourcing/IEventNotifier.cs +++ /dev/null @@ -1,22 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -#pragma warning disable MA0048 // File name must match type name - -namespace Squidex.Infrastructure.EventSourcing; - -public interface IEventNotifier -{ - void NotifyEventsStored(string streamName); -} - -public sealed class NoopEventNotifier : IEventNotifier -{ - public void NotifyEventsStored(string streamName) - { - } -} diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs b/backend/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs index 994409a60..b957ec631 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs @@ -14,7 +14,7 @@ public interface IEventStore Task> QueryReverseAsync(string streamName, int take = int.MaxValue, CancellationToken ct = default); - Task> QueryAsync(string streamName, long streamPosition = 0, + Task> QueryAsync(string streamName, long afterStreamPosition = EtagVersion.Empty, CancellationToken ct = default); IAsyncEnumerable QueryAllReverseAsync(string? streamFilter = null, Instant timestamp = default, int take = int.MaxValue, @@ -50,7 +50,7 @@ public interface IEventStore foreach (var streamName in streamNames) { - result[streamName] = await QueryAsync(streamName, 0, ct); + result[streamName] = await QueryAsync(streamName, EtagVersion.Empty, ct); } return result; diff --git a/backend/src/Squidex.Infrastructure/Queries/PropertyPath.cs b/backend/src/Squidex.Infrastructure/Queries/PropertyPath.cs index f44bd7ee8..3ee072d8a 100644 --- a/backend/src/Squidex.Infrastructure/Queries/PropertyPath.cs +++ b/backend/src/Squidex.Infrastructure/Queries/PropertyPath.cs @@ -9,7 +9,7 @@ using Squidex.Infrastructure.Collections; namespace Squidex.Infrastructure.Queries; -public sealed class PropertyPath : ReadonlyList +public sealed class PropertyPath : ReadonlyList, IEquatable { private static readonly char[] Separators = { '.', '/' }; @@ -83,6 +83,11 @@ public sealed class PropertyPath : ReadonlyList return Create(path); } + public bool Equals(string? other) + { + return string.Equals(ToString(), other, StringComparison.Ordinal); + } + public override string ToString() { return string.Join(".", this); diff --git a/backend/src/Squidex.Infrastructure/Queries/SortNode.cs b/backend/src/Squidex.Infrastructure/Queries/SortNode.cs index 76bf80a4c..dcc8e168e 100644 --- a/backend/src/Squidex.Infrastructure/Queries/SortNode.cs +++ b/backend/src/Squidex.Infrastructure/Queries/SortNode.cs @@ -7,26 +7,12 @@ namespace Squidex.Infrastructure.Queries; -public sealed class SortNode +public sealed record SortNode(PropertyPath Path, SortOrder Order) { - public PropertyPath Path { get; } - - public SortOrder Order { get; set; } - - public SortNode(PropertyPath path, SortOrder order) - { - Guard.NotNull(path); - Guard.Enum(order); - - Path = path; - - Order = order; - } - public override string ToString() { var path = string.Join(".", Path); return $"{path} {Order}"; } -} \ No newline at end of file +} diff --git a/backend/src/Squidex.Infrastructure/States/Persistence.cs b/backend/src/Squidex.Infrastructure/States/Persistence.cs index 2d0ada6d3..8e2fcc798 100644 --- a/backend/src/Squidex.Infrastructure/States/Persistence.cs +++ b/backend/src/Squidex.Infrastructure/States/Persistence.cs @@ -144,7 +144,7 @@ internal sealed class Persistence : IPersistence private async Task ReadEventsAsync( CancellationToken ct) { - var events = await eventStore.QueryAsync(streamName.Value, versionEvents + 1, ct); + var events = await eventStore.QueryAsync(streamName.Value, versionEvents, ct); var isStopped = false; diff --git a/backend/src/Squidex/Config/Domain/EventSourcingServices.cs b/backend/src/Squidex/Config/Domain/EventSourcingServices.cs index 4a01d5898..c95d3e952 100644 --- a/backend/src/Squidex/Config/Domain/EventSourcingServices.cs +++ b/backend/src/Squidex/Config/Domain/EventSourcingServices.cs @@ -30,7 +30,7 @@ public static class EventSourcingServices var mongoClient = StoreServices.GetMongoClient(mongoConfiguration); var mongoDatabase = mongoClient.GetDatabase(mongoDatabaseName); - return new MongoEventStore(mongoDatabase, c.GetRequiredService()); + return new MongoEventStore(mongoDatabase); }) .As(); }, @@ -61,9 +61,6 @@ public static class EventSourcingServices services.AddSingletonAs() .As(); - services.AddSingletonAs() - .As(); - services.AddSingleton>( sb => c => ActivatorUtilities.CreateInstance(sb, c)); } diff --git a/backend/src/Squidex/Config/Domain/StoreServices.cs b/backend/src/Squidex/Config/Domain/StoreServices.cs index 2faa1b69f..4d1795b37 100644 --- a/backend/src/Squidex/Config/Domain/StoreServices.cs +++ b/backend/src/Squidex/Config/Domain/StoreServices.cs @@ -13,7 +13,6 @@ using Migrations.Migrations.MongoDb; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.Core.Extensions.DiagnosticSources; -using MongoDB.Driver.Linq; using Squidex.Assets; using Squidex.Domain.Apps.Entities; using Squidex.Domain.Apps.Entities.Apps.DomainObject; diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/TestHelpers/TestUtils.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/TestHelpers/TestUtils.cs index fa540c35b..6dd880314 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/TestHelpers/TestUtils.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/TestHelpers/TestUtils.cs @@ -13,7 +13,6 @@ using System.Text.Json.Serialization; using MongoDB.Bson.IO; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Bson.Serialization.Serializers; using NetTopologySuite.IO.Converters; using NodaTime; using NodaTime.Serialization.SystemTextJson; @@ -63,9 +62,7 @@ public static class TestUtils public static void SetupBson() { - // Allow all types, independent from the actual assembly. - BsonSerializer.TryRegisterSerializer(new ObjectSerializer(type => true)); - + BsonDefaultConventions.Register(); BsonDomainIdSerializer.Register(); BsonEscapedDictionarySerializer.Register(); BsonEscapedDictionarySerializer.Register(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs index dcbd4e55e..332567cb8 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs @@ -7,7 +7,6 @@ using System.Globalization; using Microsoft.Extensions.DependencyInjection; -using MongoDB.Bson; using MongoDB.Driver; using NodaTime; using Squidex.Domain.Apps.Core.Assets; @@ -25,26 +24,29 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb; public sealed class AssetsQueryFixture : IAsyncLifetime { private readonly int numValues = 250; - private readonly IMongoClient mongoClient; - private readonly IMongoDatabase mongoDatabase; - public MongoAssetRepository AssetRepository { get; } - - public NamedId[] AppIds { get; } = + private readonly NamedId[] appIds = { NamedId.Of(DomainId.Create("3b5ba909-e5a5-4858-9d0d-df4ff922d452"), "my-app1"), NamedId.Of(DomainId.Create("4b3672c1-97c6-4e0b-a067-71e9e9a29db9"), "my-app1") }; + public IMongoDatabase Database { get; } + + public MongoAssetRepository AssetRepository { get; } + public AssetsQueryFixture() { BsonJsonConvention.Register(TestUtils.DefaultOptions()); - mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongodb:configuration"]); - mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); + var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]); + var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); + + Database = mongoDatabase; var services = new ServiceCollection() + .AddSingleton() .AddSingleton(mongoClient) .AddSingleton(mongoDatabase) .AddLogging() @@ -53,23 +55,22 @@ public sealed class AssetsQueryFixture : IAsyncLifetime AssetRepository = services.GetRequiredService(); } - public Task DisposeAsync() - { - return Task.CompletedTask; - } - public async Task InitializeAsync() { await AssetRepository.InitializeAsync(default); await CreateDataAsync(default); - await ClearProfileAsync(default); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; } private async Task CreateDataAsync( CancellationToken ct) { - if (await AssetRepository.StreamAll(AppIds[0].Id, ct).AnyAsync(ct)) + if (await AssetRepository.StreamAll(appIds[0].Id, ct).AnyAsync(ct)) { return; } @@ -88,16 +89,14 @@ public sealed class AssetsQueryFixture : IAsyncLifetime var store = (ISnapshotStore)AssetRepository; await store.WriteManyAsync(batch, ct); - batch.Clear(); } } - var now = SystemClock.Instance.GetCurrentInstant(); + var created = SystemClock.Instance.GetCurrentInstant(); + var createdBy = RefToken.User("1"); - var user = RefToken.User("1"); - - foreach (var appId in AppIds) + foreach (var appId in appIds) { for (var i = 0; i < numValues; i++) { @@ -111,13 +110,13 @@ public sealed class AssetsQueryFixture : IAsyncLifetime { Id = DomainId.NewGuid(), AppId = appId, - Created = now, - CreatedBy = user, + Created = created, + CreatedBy = createdBy, FileHash = fileName, FileName = fileName, FileSize = 1024, - LastModified = now, - LastModifiedBy = user, + LastModified = created, + LastModifiedBy = createdBy, IsDeleted = false, IsProtected = false, Metadata = new AssetMetadata @@ -139,17 +138,9 @@ public sealed class AssetsQueryFixture : IAsyncLifetime await ExecuteBatchAsync(null); } - private async Task ClearProfileAsync( - CancellationToken ct) - { - await mongoDatabase.RunCommandAsync("{ profile : 0 }", cancellationToken: ct); - await mongoDatabase.DropCollectionAsync("system.profile", ct); - await mongoDatabase.RunCommandAsync("{ profile : 2 }", cancellationToken: ct); - } - public DomainId RandomAppId() { - return AppIds[Random.Shared.Next(AppIds.Length)].Id; + return appIds[Random.Shared.Next(appIds.Length)].Id; } public string RandomValue() diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs index d9029baa4..71ed4a433 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs @@ -5,7 +5,9 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using Amazon.Runtime; using Squidex.Infrastructure; +using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.Queries; using F = Squidex.Infrastructure.Queries.ClrFilter; @@ -14,13 +16,32 @@ using F = Squidex.Infrastructure.Queries.ClrFilter; namespace Squidex.Domain.Apps.Entities.Assets.MongoDb; [Trait("Category", "Dependencies")] -public class AssetsQueryIntegrationTests : IClassFixture +public class AssetsQueryIntegrationTests : IClassFixture, IAsyncLifetime { + private readonly ProfilerCollection profiler; + public AssetsQueryFixture _ { get; } public AssetsQueryIntegrationTests(AssetsQueryFixture fixture) { _ = fixture; + + profiler = new ProfilerCollection(_.Database); + } + + public Task InitializeAsync() + { + return profiler.ClearAsync(); + } + + public async Task DisposeAsync() + { + var queries = await profiler.GetQueriesAsync(_.AssetRepository.GetInternalCollection().CollectionNamespace.CollectionName); + + Assert.All(queries, query => + { + Assert.Equal(query.NumDocuments, query.DocsExamined); + }); } [Fact] @@ -46,7 +67,7 @@ public class AssetsQueryIntegrationTests : IClassFixture } [Fact] - public async Task Should_verify_ids() + public async Task Should_query_ids() { var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet(); @@ -157,18 +178,22 @@ public class AssetsQueryIntegrationTests : IClassFixture yield return new object?[] { DomainId.Empty }; } - private async Task> QueryAsync(DomainId? parentId, ClrQuery clrQuery) + private async Task> QueryAsync(DomainId? parentId, ClrQuery clrQuery, + int top = 1000, + int skip = 100) { - clrQuery.Top = 1000; - clrQuery.Skip = 100; + clrQuery.Top = top; + clrQuery.Skip = skip; + clrQuery.Sort ??= new List(); + + if (clrQuery.Sort.Count == 0) + { + clrQuery.Sort.Add(new SortNode("lastModified", SortOrder.Descending)); + } - if (clrQuery.Sort == null || clrQuery.Sort.Count == 0) + if (!clrQuery.Sort.Exists(x => x.Path.Equals("id"))) { - clrQuery.Sort = new List - { - new SortNode("LastModified", SortOrder.Descending), - new SortNode("Id", SortOrder.Descending) - }; + clrQuery.Sort.Add(new SortNode("id", SortOrder.Ascending)); } var q = diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryDedicatedIntegrationTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryDedicatedIntegrationTests.cs index fcc2e440f..5acf85850 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryDedicatedIntegrationTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryDedicatedIntegrationTests.cs @@ -8,9 +8,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb; [Trait("Category", "Dependencies")] -public class ContentsQueryDedicatedIntegrationTests : ContentsQueryTestsBase, IClassFixture +public class ContentsQueryDedicatedIntegrationTests : ContentsQueryTestsBase, IClassFixture { - public ContentsQueryDedicatedIntegrationTests(ContentsQueryDedicatedFixture fixture) + public ContentsQueryDedicatedIntegrationTests(ContentsQueryFixture_Dedicated fixture) : base(fixture) { } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs index 9a71d4cf1..afeac8a62 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs @@ -30,27 +30,27 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Contents.MongoDb; -public sealed class ContentsQueryFixture : ContentsQueryFixtureBase +public sealed class ContentsQueryFixture_Default : ContentsQueryFixture { - public ContentsQueryFixture() + public ContentsQueryFixture_Default() : base(false) { } } -public sealed class ContentsQueryDedicatedFixture : ContentsQueryFixtureBase +public sealed class ContentsQueryFixture_Dedicated : ContentsQueryFixture { - public ContentsQueryDedicatedFixture() + public ContentsQueryFixture_Dedicated() : base(true) { } } -public abstract class ContentsQueryFixtureBase : IAsyncLifetime +public abstract class ContentsQueryFixture : IAsyncLifetime { private readonly int numValues = 10000; - private readonly IMongoClient mongoClient; - private readonly IMongoDatabase mongoDatabase; + + public IMongoDatabase Database { get; } public MongoContentRepository ContentRepository { get; } @@ -69,36 +69,38 @@ public abstract class ContentsQueryFixtureBase : IAsyncLifetime NamedId.Of(DomainId.Create("741e902c-fdfa-41ad-8e5a-b7cb9d6e3d94"), "my-schema5") }; - protected ContentsQueryFixtureBase(bool dedicatedCollections) + protected ContentsQueryFixture(bool selfHosting) { BsonJsonConvention.Register(TestUtils.DefaultOptions()); - mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongodb:configuration"]); - mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); + var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]); + var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); + + Database = mongoDatabase; var services = new ServiceCollection() - .AddSingleton(Options.Create(new ContentOptions { OptimizeForSelfHosting = dedicatedCollections })) + .AddSingleton(Options.Create(new ContentOptions { OptimizeForSelfHosting = selfHosting })) .AddSingleton(CreateAppProvider()) .AddSingleton(mongoClient) .AddSingleton(mongoDatabase) + .AddSingleton() .AddLogging() .BuildServiceProvider(); ContentRepository = services.GetRequiredService(); } - public Task DisposeAsync() - { - return Task.CompletedTask; - } - public async Task InitializeAsync() { await ContentRepository.InitializeAsync(default); await CreateDataAsync(default); - await ClearProfilerAsync(default); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; } private async Task CreateDataAsync( @@ -128,9 +130,8 @@ public abstract class ContentsQueryFixtureBase : IAsyncLifetime } } - var now = SystemClock.Instance.GetCurrentInstant(); - - var user = RefToken.User("1"); + var created = SystemClock.Instance.GetCurrentInstant(); + var createdBy = RefToken.User("1"); foreach (var appId in AppIds) { @@ -151,12 +152,12 @@ public abstract class ContentsQueryFixtureBase : IAsyncLifetime { Id = DomainId.NewGuid(), AppId = appId, - Created = now, - CreatedBy = user, + Created = created, + CreatedBy = createdBy, CurrentVersion = new ContentVersion(Status.Published, data), IsDeleted = false, - LastModified = now, - LastModifiedBy = user, + LastModified = created, + LastModifiedBy = createdBy, SchemaId = schemaId }; @@ -168,26 +169,6 @@ public abstract class ContentsQueryFixtureBase : IAsyncLifetime await ExecuteBatchAsync(null); } - private async Task ClearProfilerAsync( - CancellationToken ct) - { - var prefix = mongoDatabase.DatabaseNamespace.DatabaseName; - - foreach (var databaseName in await (await mongoClient.ListDatabaseNamesAsync(ct)).ToListAsync(ct)) - { - if (!databaseName.StartsWith(prefix, StringComparison.Ordinal)) - { - continue; - } - - var database = mongoClient.GetDatabase(databaseName); - - await database.RunCommandAsync("{ profile : 0 }", cancellationToken: ct); - await database.DropCollectionAsync("system.profile", ct); - await database.RunCommandAsync("{ profile : 2 }", cancellationToken: ct); - } - } - private static IAppProvider CreateAppProvider() { var appProvider = A.Fake(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs index ed85725ab..d531a13eb 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs @@ -8,9 +8,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb; [Trait("Category", "Dependencies")] -public class ContentsQueryIntegrationTests : ContentsQueryTestsBase, IClassFixture +public class ContentsQueryIntegrationTests : ContentsQueryTestsBase, IClassFixture { - public ContentsQueryIntegrationTests(ContentsQueryFixture fixture) + public ContentsQueryIntegrationTests(ContentsQueryFixture_Default fixture) : base(fixture) { } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTestsBase.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTestsBase.cs index 818e94212..33dcd7de0 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTestsBase.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTestsBase.cs @@ -18,9 +18,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb; public abstract class ContentsQueryTestsBase { - public ContentsQueryFixtureBase _ { get; } + public ContentsQueryFixture _ { get; } - protected ContentsQueryTestsBase(ContentsQueryFixtureBase fixture) + protected ContentsQueryTestsBase(ContentsQueryFixture fixture) { _ = fixture; } @@ -196,27 +196,22 @@ public abstract class ContentsQueryTestsBase private async Task> QueryAsync(IContentRepository contentRepository, ClrQuery clrQuery, - int take = 1000, + int top = 1000, int skip = 100, DomainId reference = default) { - if (clrQuery.Take == long.MaxValue) - { - clrQuery.Take = take; - } + clrQuery.Take = top; + clrQuery.Skip = skip; + clrQuery.Sort ??= new List(); - if (clrQuery.Skip == 0) + if (clrQuery.Sort.Count == 0) { - clrQuery.Skip = skip; + clrQuery.Sort.Add(new SortNode("lastModified", SortOrder.Descending)); } - if (clrQuery.Sort == null || clrQuery.Sort.Count == 0) + if (!clrQuery.Sort.Exists(x => x.Path.Equals("id"))) { - clrQuery.Sort = new List - { - new SortNode("LastModified", SortOrder.Descending), - new SortNode("Id", SortOrder.Ascending) - }; + clrQuery.Sort.Add(new SortNode("id", SortOrder.Ascending)); } var q = diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs index a051349b5..a1730eddb 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs @@ -9,7 +9,6 @@ using System.Net; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using MongoDB.Driver; using Squidex.Domain.Apps.Core.TestHelpers; using Squidex.Domain.Apps.Entities.MongoDb.Text; using Squidex.Domain.Apps.Entities.TestHelpers; diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs index edba7a781..93d740214 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using MongoDB.Driver; using Squidex.Domain.Apps.Core.TestHelpers; using Squidex.Domain.Apps.Entities.MongoDb.Text; using Squidex.Domain.Apps.Entities.TestHelpers; @@ -21,7 +20,7 @@ public sealed class MongoTextIndexFixture : IAsyncLifetime { TestUtils.SetupBson(); - var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongodb:configuration"]); + var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]); var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); Index = new MongoTextIndex(mongoDatabase); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs index f604961c1..18869e3bf 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using MongoDB.Driver; using Squidex.Domain.Apps.Core.TestHelpers; using Squidex.Domain.Apps.Entities.MongoDb.Schemas; using Squidex.Domain.Apps.Entities.TestHelpers; @@ -21,7 +20,7 @@ public sealed class SchemasHashFixture { TestUtils.SetupBson(); - var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongodb:configuration"]); + var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]); var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); var schemasHash = new MongoSchemasHash(mongoDatabase); diff --git a/backend/tests/Squidex.Domain.Users.Tests/DefaultUserResolverTests.cs b/backend/tests/Squidex.Domain.Users.Tests/DefaultUserResolverTests.cs index 54ed27bd3..ddd86bffe 100644 --- a/backend/tests/Squidex.Domain.Users.Tests/DefaultUserResolverTests.cs +++ b/backend/tests/Squidex.Domain.Users.Tests/DefaultUserResolverTests.cs @@ -83,7 +83,7 @@ public class DefaultUserResolverTests await sut.SetClaimAsync(id, "my-claim", "my-value", false, ct); A.CallTo(() => userService.UpdateAsync(id, - A.That.Matches(x => x.CustomClaims!.Any(y => y.Type == "my-claim" && y.Value == "my-value")), false, ct)) + A.That.Matches(x => x.CustomClaims!.Exists(y => y.Type == "my-claim" && y.Value == "my-value")), false, ct)) .MustHaveHappened(); } @@ -95,7 +95,7 @@ public class DefaultUserResolverTests await sut.SetClaimAsync(id, "my-claim", "my-value", true, ct); A.CallTo(() => userService.UpdateAsync(id, - A.That.Matches(x => x.CustomClaims!.Any(y => y.Type == "my-claim" && y.Value == "my-value")), true, ct)) + A.That.Matches(x => x.CustomClaims!.Exists(y => y.Type == "my-claim" && y.Value == "my-value")), true, ct)) .MustHaveHappened(); } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorIntegrationTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorIntegrationTests.cs index 77070f160..a5922aad1 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorIntegrationTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorIntegrationTests.cs @@ -6,7 +6,6 @@ // ========================================================================== using Microsoft.Extensions.DependencyInjection; -using MongoDB.Driver; using Squidex.Caching; using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.Reflection; @@ -62,7 +61,7 @@ public abstract class EventConsumerProcessorIntegrationTests var eventConsumer = new EventConsumer(); - var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongodb:configuration"]); + var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configurationDirect"]); var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); var typeRegistry = new TypeRegistry().Add("MyEvent"); @@ -130,7 +129,9 @@ public abstract class EventConsumerProcessorIntegrationTests { while (!cts.IsCancellationRequested && eventConsumer.Events.Count < expectedEvents) { +#pragma warning disable MA0040 // Forward the CancellationToken parameter to methods that take one await Task.Delay(100); +#pragma warning restore MA0040 // Forward the CancellationToken parameter to methods that take one } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/MongoEventConsumerProcessorIntegrationTests_Direct.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/MongoEventConsumerProcessorIntegrationTests_Direct.cs index befc06a1e..c39c127ab 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/MongoEventConsumerProcessorIntegrationTests_Direct.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/MongoEventConsumerProcessorIntegrationTests_Direct.cs @@ -10,11 +10,11 @@ namespace Squidex.Infrastructure.EventSourcing.Consume; [Trait("Category", "Dependencies")] -public class MongoEventConsumerProcessorIntegrationTests_Direct : EventConsumerProcessorIntegrationTests, IClassFixture +public class MongoEventConsumerProcessorIntegrationTests_Direct : EventConsumerProcessorIntegrationTests, IClassFixture { public MongoEventStoreFixture _ { get; } - public MongoEventConsumerProcessorIntegrationTests_Direct(MongoEventStoreDirectFixture fixture) + public MongoEventConsumerProcessorIntegrationTests_Direct(MongoEventStoreFixture_Direct fixture) { _ = fixture; } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs index d30292c77..2a4bf7d0b 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs @@ -6,6 +6,7 @@ // ========================================================================== using System.Globalization; +using System.Text.RegularExpressions; namespace Squidex.Infrastructure.EventSourcing; @@ -16,7 +17,7 @@ public abstract class EventStoreTests where T : IEventStore public sealed class EventSubscriber : IEventSubscriber { - public List Events { get; } = new List(); + public List LastEvents { get; } = new List(); public string LastPosition { get; set; } @@ -36,9 +37,7 @@ public abstract class EventStoreTests where T : IEventStore public ValueTask OnNextAsync(IEventSubscription subscription, StoredEvent @event) { LastPosition = @event.EventPosition; - - Events.Add(@event); - + LastEvents.Add(@event); return default; } } @@ -255,33 +254,6 @@ public abstract class EventStoreTests where T : IEventStore Assert.Equal(numEvents * numTasks, readEvents?.Count); } - [Fact] - public async Task Should_read_events_from_offset() - { - var streamName = $"test-{Guid.NewGuid()}"; - - var commit = new[] - { - CreateEventData(1), - CreateEventData(2) - }; - - await Sut.AppendAsync(Guid.NewGuid(), streamName, EtagVersion.Any, commit); - - var firstRead = await QueryAsync(streamName); - - var readEvents1 = await QueryAsync(streamName, 1); - var readEvents2 = await QueryAllAsync(streamName, firstRead[0].EventPosition); - - var expected = new[] - { - new StoredEvent(streamName, "Position", 1, commit[1]) - }; - - ShouldBeEquivalentTo(readEvents1, expected); - ShouldBeEquivalentTo(readEvents2, expected); - } - [Fact] public async Task Should_read_multiple_streams() { @@ -322,31 +294,42 @@ public abstract class EventStoreTests where T : IEventStore } [Theory] + [InlineData(1, 30)] [InlineData(5, 30)] [InlineData(5, 300)] - public async Task Should_read_latest_events(int commitSize, int count) + [InlineData(5, 3000)] + public async Task Should_read_events_from_offset(int commits, int count) { var streamName = $"test-{Guid.NewGuid()}"; - var events = new List(); + var eventsWritten = await AppendEventsAsync(streamName, count, commits); - for (var i = 0; i < count; i++) - { - events.Add(CreateEventData(i)); - } + var readEvents0 = await QueryAsync(streamName); + var readEvents1 = await QueryAsync(streamName, count - 2); + var readEvents2 = await QueryAllAsync(streamName, readEvents0[^2].EventPosition); - for (var i = 0; i < events.Count / commitSize; i++) + var expected = new[] { - var commit = events.Skip(i * commitSize).Take(commitSize).ToArray(); + new StoredEvent(streamName, "Position", count - 1, eventsWritten[^1]) + }; - await Sut.AppendAsync(Guid.NewGuid(), streamName, EtagVersion.Any, commit); - } + ShouldBeEquivalentTo(readEvents1, expected); + ShouldBeEquivalentTo(readEvents2, expected); + } + + [Theory] + [InlineData(5, 30)] + [InlineData(5, 300)] + public async Task Should_read_reverse(int commits, int count) + { + var streamName = $"test-{Guid.NewGuid()}"; - var allExpected = events.Select((x, i) => new StoredEvent(streamName, "Position", i, events[i])).ToArray(); + var eventsWritten = await AppendEventsAsync(streamName, count, commits); + var eventsStored = eventsWritten.Select((x, i) => new StoredEvent(streamName, "Position", i, x)).ToArray(); for (var take = 0; take < count; take += count / 10) { - var eventsExpected = allExpected.TakeLast(take).ToArray(); + var eventsExpected = eventsStored.TakeLast(take).ToArray(); var eventsQueried = await Sut.QueryReverseAsync(streamName, take); ShouldBeEquivalentTo(eventsQueried, eventsExpected); @@ -356,32 +339,61 @@ public abstract class EventStoreTests where T : IEventStore [Theory] [InlineData(5, 30)] [InlineData(5, 300)] - public async Task Should_read_reverse(int commitSize, int count) + [InlineData(5, 3000)] + public async Task Should_read_all_reverse_by_name(int commits, int count) { var streamName = $"test-{Guid.NewGuid()}"; + var streamFilter = streamName; - var events = new List(); + var eventsWritten = await AppendEventsAsync(streamName, count, commits); + var eventsStored = eventsWritten.Select((x, i) => new StoredEvent(streamName, "Position", i, x)).ToArray(); - for (var i = 0; i < count; i++) + for (var take = 0; take < count; take += count / 10) { - events.Add(CreateEventData(i)); + var eventsExpected = eventsStored.Reverse().Take(take).ToArray(); + var eventsQueried = await Sut.QueryAllReverseAsync(streamName, default, take).ToArrayAsync(); + + ShouldBeEquivalentTo(eventsQueried, eventsExpected); } + } + + [Theory] + [InlineData(5, 30)] + [InlineData(5, 300)] + [InlineData(5, 3000)] + public async Task Should_read_all_reverse_by_filter(int commits, int count) + { + var streamName = $"test-{Guid.NewGuid()}-suffix"; + var streamFilter = $"^{Regex.Escape(streamName)}"; + + var eventsWritten = await AppendEventsAsync(streamName, count, commits); + var eventsStored = eventsWritten.Select((x, i) => new StoredEvent(streamName, "Position", i, x)).ToArray(); - for (var i = 0; i < events.Count / commitSize; i++) + for (var take = 0; take < count; take += count / 10) { - var commit = events.Skip(i * commitSize).Take(commitSize).ToArray(); + var eventsExpected = eventsStored.Reverse().Take(take).ToArray(); + var eventsQueried = await Sut.QueryAllReverseAsync(streamFilter, default, take).ToArrayAsync(); - await Sut.AppendAsync(Guid.NewGuid(), streamName, EtagVersion.Any, commit); + ShouldBeEquivalentTo(eventsQueried, eventsExpected); } + } + + [Theory] + [InlineData(5, 30)] + [InlineData(5, 300)] + [InlineData(5, 3000)] + public async Task Should_read_all_reverse(int commits, int count) + { + var streamName = $"test-{Guid.NewGuid()}-suffix"; + var streamFilter = ".*"; - var allExpected = events.Select((x, i) => new StoredEvent(streamName, "Position", i, events[i])).ToArray(); + await AppendEventsAsync(streamName, count, commits); for (var take = 0; take < count; take += count / 10) { - var eventsExpected = allExpected.Reverse().Take(take).ToArray(); - var eventsQueried = await Sut.QueryAllReverseAsync(streamName, default, take).ToArrayAsync(); + var eventsQueried = await Sut.QueryAllReverseAsync(streamFilter, default, take).ToArrayAsync(); - ShouldBeEquivalentTo(eventsQueried, eventsExpected); + Assert.Equal(take, eventsQueried.Length); } } @@ -390,13 +402,7 @@ public abstract class EventStoreTests where T : IEventStore { var streamName = $"test-{Guid.NewGuid()}"; - var events = new[] - { - CreateEventData(1), - CreateEventData(2) - }; - - await Sut.AppendAsync(Guid.NewGuid(), streamName, EtagVersion.Any, events); + await AppendEventsAsync(streamName, 2, 1); IReadOnlyList? readEvents = null; @@ -423,13 +429,7 @@ public abstract class EventStoreTests where T : IEventStore { var streamName = $"test-{Guid.NewGuid()}"; - var events = new[] - { - CreateEventData(1), - CreateEventData(2) - }; - - await Sut.AppendAsync(Guid.NewGuid(), streamName, EtagVersion.Any, events); + await AppendEventsAsync(streamName, 2, 1); IReadOnlyList? readEvents = null; @@ -451,9 +451,14 @@ public abstract class EventStoreTests where T : IEventStore Assert.Empty(readEvents!); } - private Task> QueryAsync(string streamName, long position = EtagVersion.Any) + private async Task> QueryAsync(string streamName, long position = EtagVersion.Any) + { + return await Sut.QueryAsync(streamName, position); + } + + private async Task?> QueryAllAsync(string? streamFilter = null, string? position = null) { - return Sut.QueryAsync(streamName, position); + return await Sut.QueryAllAsync(streamFilter, position).ToListAsync(); } private static EventData CreateEventData(int i) @@ -466,18 +471,6 @@ public abstract class EventStoreTests where T : IEventStore return new EventData($"Type{i}", headers, i.ToString(CultureInfo.InvariantCulture)); } - private async Task?> QueryAllAsync(string? streamFilter = null, string? position = null) - { - var readEvents = new List(); - - await foreach (var storedEvent in Sut.QueryAllAsync(streamFilter, position)) - { - readEvents.Add(storedEvent); - } - - return readEvents; - } - private async Task?> QueryWithSubscriptionAsync(string streamFilter, Func? subscriptionRunning = null, bool fromBeginning = false) { @@ -501,11 +494,11 @@ public abstract class EventStoreTests where T : IEventStore await Task.Delay(2000, cts.Token); - if (subscriber.Events.Count > 0) + if (subscriber.LastEvents.Count > 0) { subscriptionPosition = subscriber.LastPosition; - return subscriber.Events; + return subscriber.LastEvents; } } @@ -520,8 +513,27 @@ public abstract class EventStoreTests where T : IEventStore } } + private async Task> AppendEventsAsync(string streamName, int count, int commits = 1) + { + var events = new List(); + + for (var i = 0; i < count; i++) + { + events.Add(CreateEventData(i)); + } + + for (var i = 0; i < events.Count / commits; i++) + { + var commit = events.Skip(i * commits).Take(commits).ToArray(); + + await Sut.AppendAsync(Guid.NewGuid(), streamName, EtagVersion.Any, commit); + } + + return events; + } + private static void ShouldBeEquivalentTo(IEnumerable? actual, params StoredEvent[] expected) { - actual.Should().BeEquivalentTo(expected, opts => opts.ComparingByMembers().Including(x => x.EventStreamNumber)); + actual.Should().BeEquivalentTo(expected, opts => opts.ComparingByMembers().Excluding(x => x.EventPosition)); } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs index 94734b13d..5825a494f 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs @@ -6,7 +6,6 @@ // ========================================================================== using MongoDB.Driver; -using MongoDB.Driver.Linq; using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.TestHelpers; @@ -14,22 +13,41 @@ using Squidex.Infrastructure.TestHelpers; namespace Squidex.Infrastructure.EventSourcing; -public abstract class MongoEventStoreFixture : IAsyncLifetime +public sealed class MongoEventStoreFixture_Direct : MongoEventStoreFixture { - private readonly IMongoClient mongoClient; - private readonly IMongoDatabase mongoDatabase; - private readonly IEventNotifier notifier = A.Fake(); + public MongoEventStoreFixture_Direct() + : base(TestConfig.Configuration["mongoDb:configurationDirect"]!) + { + } +} + +public sealed class MongoEventStoreFixture_Replica : MongoEventStoreFixture +{ + public MongoEventStoreFixture_Replica() + : base(TestConfig.Configuration["mongoDb:configurationReplica"]!) + { + } +} +public abstract class MongoEventStoreFixture : IAsyncLifetime +{ public MongoEventStore EventStore { get; } + public IMongoDatabase Database { get; } + + static MongoEventStoreFixture() + { + BsonJsonConvention.Register(TestUtils.DefaultOptions()); + } + protected MongoEventStoreFixture(string connectionString) { - mongoClient = MongoClientFactory.Create(connectionString); - mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); + var mongoClient = MongoClientFactory.Create(connectionString); + var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); - BsonJsonConvention.Register(TestUtils.DefaultOptions()); + Database = mongoDatabase; - EventStore = new MongoEventStore(mongoDatabase, notifier); + EventStore = new MongoEventStore(mongoDatabase); } public Task InitializeAsync() @@ -39,22 +57,6 @@ public abstract class MongoEventStoreFixture : IAsyncLifetime public Task DisposeAsync() { - return mongoClient.DropDatabaseAsync(mongoDatabase.DatabaseNamespace.DatabaseName); - } -} - -public sealed class MongoEventStoreDirectFixture : MongoEventStoreFixture -{ - public MongoEventStoreDirectFixture() - : base(TestConfig.Configuration["mongodb:configuration"]!) - { - } -} - -public sealed class MongoEventStoreReplicaSetFixture : MongoEventStoreFixture -{ - public MongoEventStoreReplicaSetFixture() - : base(TestConfig.Configuration["mongodb:configurationReplica"]!) - { + return Task.CompletedTask; } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests.cs new file mode 100644 index 000000000..4f18b632d --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests.cs @@ -0,0 +1,67 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +#pragma warning disable SA1300 // Element should begin with upper-case letter +#pragma warning disable MA0048 // File name must match type name + +using Squidex.Infrastructure.MongoDb; + +namespace Squidex.Infrastructure.EventSourcing; + +public abstract class MongoEventStoreTests : EventStoreTests, IAsyncLifetime +{ + private ProfilerCollection profiler; + + public MongoEventStoreFixture _ { get; } + + protected MongoEventStoreTests(MongoEventStoreFixture fixture) + { + _ = fixture; + } + + public override MongoEventStore CreateStore() + { + return _.EventStore; + } + + public Task InitializeAsync() + { + profiler = new ProfilerCollection(_.Database); + + return profiler.ClearAsync(); + } + + public async Task DisposeAsync() + { + var queries = await profiler.GetQueriesAsync("Events2"); + + Assert.All(queries, query => + { + Assert.Equal(query.NumDocuments, query.DocsExamined); + Assert.True(query.KeysExamined >= query.NumDocuments); + Assert.True(query.KeysExamined <= query.NumDocuments * 2); + }); + } +} + +[Trait("Category", "Dependencies")] +public sealed class MongoEventStoreTests_Direct : MongoEventStoreTests, IClassFixture +{ + public MongoEventStoreTests_Direct(MongoEventStoreFixture_Direct fixture) + : base(fixture) + { + } +} + +[Trait("Category", "Dependencies")] +public sealed class MongoEventStoreTests_Replica : MongoEventStoreTests, IClassFixture +{ + public MongoEventStoreTests_Replica(MongoEventStoreFixture_Replica fixture) + : base(fixture) + { + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs deleted file mode 100644 index 9a6b6d58a..000000000 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -#pragma warning disable SA1300 // Element should begin with upper-case letter - -namespace Squidex.Infrastructure.EventSourcing; - -[Trait("Category", "Dependencies")] -public class MongoEventStoreTests_Direct : EventStoreTests, IClassFixture -{ - public MongoEventStoreFixture _ { get; } - - public MongoEventStoreTests_Direct(MongoEventStoreDirectFixture fixture) - { - _ = fixture; - } - - public override MongoEventStore CreateStore() - { - return _.EventStore; - } -} diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs deleted file mode 100644 index 331fa75c3..000000000 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -#pragma warning disable SA1300 // Element should begin with upper-case letter - -namespace Squidex.Infrastructure.EventSourcing; - -[Trait("Category", "Dependencies")] -public class MongoEventStoreTests_ReplicaSet : EventStoreTests, IClassFixture -{ - public MongoEventStoreFixture _ { get; } - - public MongoEventStoreTests_ReplicaSet(MongoEventStoreReplicaSetFixture fixture) - { - _ = fixture; - } - - public override MongoEventStore CreateStore() - { - return _.EventStore; - } -} diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs index 007bf486a..999319152 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs @@ -15,7 +15,7 @@ using Squidex.Infrastructure.TestHelpers; namespace Squidex.Infrastructure.EventSourcing; [Trait("Category", "Dependencies")] -public sealed class MongoParallelInsertTests : IClassFixture +public sealed class MongoParallelInsertTests : IClassFixture { private readonly TestState state = new TestState(DomainId.Empty); private readonly IEventFormatter eventFormatter; @@ -65,7 +65,7 @@ public sealed class MongoParallelInsertTests : IClassFixture eventStore.QueryAsync(key.ToString(), 0, A._)) + A.CallTo(() => eventStore.QueryAsync(key.ToString(), -1, A._)) .Returns(new List { storedEvent }); A.CallTo(() => eventFormatter.ParseIfKnown(storedEvent)) @@ -96,17 +96,17 @@ public class PersistenceEventSourcingTests Assert.False(persistence.IsSnapshotStale); - A.CallTo(() => eventStore.QueryAsync(key.ToString(), 3, A._)) + A.CallTo(() => eventStore.QueryAsync(key.ToString(), 2, A._)) .MustHaveHappened(); } [Fact] - public async Task Should_mark_as_stale_if_snapshot_old_than_events() + public async Task Should_mark_as_stale_if_snapshot_is_older_than_events() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) .Returns(new SnapshotResult(key, "2", 1)); - SetupEventStore(3, 2, 2); + SetupEventStore(3, 2, 1); var persistedState = Save.Snapshot(string.Empty); var persistedEvents = Save.Events(); @@ -116,7 +116,7 @@ public class PersistenceEventSourcingTests Assert.True(persistence.IsSnapshotStale); - A.CallTo(() => eventStore.QueryAsync(key.ToString(), 2, A._)) + A.CallTo(() => eventStore.QueryAsync(key.ToString(), 1, A._)) .MustHaveHappened(); } @@ -126,7 +126,7 @@ public class PersistenceEventSourcingTests A.CallTo(() => snapshotStore.ReadAsync(key, A._)) .Returns(new SnapshotResult(key, "2", 2)); - SetupEventStore(3, 0, 3); + SetupEventStore(3, 0, 2); var persistedState = Save.Snapshot(string.Empty); var persistedEvents = Save.Events(); @@ -141,7 +141,7 @@ public class PersistenceEventSourcingTests A.CallTo(() => snapshotStore.ReadAsync(key, A._)) .Returns(new SnapshotResult(key, "2", 2)); - SetupEventStore(3, 4, 3); + SetupEventStore(3, 4, 2); var persistedState = Save.Snapshot(string.Empty); var persistedEvents = Save.Events(); @@ -350,17 +350,17 @@ public class PersistenceEventSourcingTests Assert.Equal(EtagVersion.Empty, persistence.Version); } - private void SetupEventStore(int count, int eventOffset = 0, int readPosition = 0) + private void SetupEventStore(int count, int eventOffset = 0, long readPosition = EtagVersion.Empty) { SetupEventStore(Enumerable.Repeat(0, count).Select(x => new MyEvent()).ToArray(), eventOffset, readPosition); } private void SetupEventStore(params MyEvent[] events) { - SetupEventStore(events, 0, 0); + SetupEventStore(events, 0, EtagVersion.Empty); } - private void SetupEventStore(MyEvent[] events, int eventOffset, int readPosition = 0) + private void SetupEventStore(MyEvent[] events, int eventOffset = 0, long readPosition = EtagVersion.Empty) { var eventsStored = new List(); diff --git a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestUtils.cs b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestUtils.cs index 2b498a77a..f02b04d32 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestUtils.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestUtils.cs @@ -12,7 +12,6 @@ using System.Text.Json.Serialization; using MongoDB.Bson.IO; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Bson.Serialization.Serializers; using NodaTime; using NodaTime.Serialization.SystemTextJson; using Squidex.Infrastructure.Json; @@ -46,9 +45,7 @@ public static class TestUtils public static void SetupBson() { - // Allow all types, independent from the actual assembly. - BsonSerializer.TryRegisterSerializer(new ObjectSerializer(type => true)); - + BsonDefaultConventions.Register(); BsonDomainIdSerializer.Register(); BsonEscapedDictionarySerializer.Register(); BsonInstantSerializer.Register(); diff --git a/backend/tests/Squidex.Infrastructure.Tests/appSettings.json b/backend/tests/Squidex.Infrastructure.Tests/appSettings.json index 40ea78339..5ec25a0dd 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/appSettings.json +++ b/backend/tests/Squidex.Infrastructure.Tests/appSettings.json @@ -3,7 +3,7 @@ // The connection string to your Mongo Server. // // Read More: https://docs.mongodb.com/manual/reference/connection-string/ - "configuration": "mongodb://localhost", + "configurationDirect": "mongodb://localhost", "configurationReplica": "mongodb://localhost", // The name of the event store database. diff --git a/backend/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs b/backend/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs index d3424b357..d813e548d 100644 --- a/backend/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs +++ b/backend/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs @@ -119,7 +119,7 @@ public class AppResolverTests : GivenContext Assert.Same(App, httpContext.Context().App); Assert.True(user.Claims.Any()); Assert.True(permissions.Count < 3); - Assert.True(permissions.All(x => x.Value.StartsWith($"squidex.apps.{AppId.Name}", StringComparison.OrdinalIgnoreCase))); + Assert.True(permissions.TrueForAll(x => x.Value.StartsWith($"squidex.apps.{AppId.Name}", StringComparison.OrdinalIgnoreCase))); Assert.True(isNextCalled); } @@ -143,7 +143,7 @@ public class AppResolverTests : GivenContext Assert.Same(App, httpContext.Context().App); Assert.True(user.Claims.Count() > 2); Assert.True(permissions.Count < 3); - Assert.True(permissions.All(x => x.Value.StartsWith($"squidex.apps.{AppId.Name}", StringComparison.OrdinalIgnoreCase))); + Assert.True(permissions.TrueForAll(x => x.Value.StartsWith($"squidex.apps.{AppId.Name}", StringComparison.OrdinalIgnoreCase))); Assert.True(isNextCalled); } @@ -168,7 +168,7 @@ public class AppResolverTests : GivenContext Assert.Same(App, httpContext.Context().App); Assert.True(user.Claims.Count() > 2); Assert.True(permissions.Count > 10); - Assert.True(permissions.All(x => x.Value.StartsWith($"squidex.apps.{AppId.Name}", StringComparison.OrdinalIgnoreCase))); + Assert.True(permissions.TrueForAll(x => x.Value.StartsWith($"squidex.apps.{AppId.Name}", StringComparison.OrdinalIgnoreCase))); Assert.True(isNextCalled); } diff --git a/backend/tests/Squidex.Web.Tests/Pipeline/TeamResolverTests.cs b/backend/tests/Squidex.Web.Tests/Pipeline/TeamResolverTests.cs index 872b5fd3f..25319d746 100644 --- a/backend/tests/Squidex.Web.Tests/Pipeline/TeamResolverTests.cs +++ b/backend/tests/Squidex.Web.Tests/Pipeline/TeamResolverTests.cs @@ -119,7 +119,7 @@ public class TeamResolverTests : GivenContext Assert.Same(Team, httpContext.Features.Get()!.Team); Assert.True(user.Claims.Any()); Assert.True(permissions.Count < 3); - Assert.True(permissions.All(x => x.Value.StartsWith($"squidex.teams.{TeamId}", StringComparison.OrdinalIgnoreCase))); + Assert.True(permissions.TrueForAll(x => x.Value.StartsWith($"squidex.teams.{TeamId}", StringComparison.OrdinalIgnoreCase))); Assert.True(isNextCalled); } @@ -143,7 +143,7 @@ public class TeamResolverTests : GivenContext Assert.Same(Team, httpContext.Features.Get()!.Team); Assert.True(user.Claims.Count() > 2); Assert.True(permissions.Count < 3); - Assert.True(permissions.All(x => x.Value.StartsWith($"squidex.teams.{TeamId}", StringComparison.OrdinalIgnoreCase))); + Assert.True(permissions.TrueForAll(x => x.Value.StartsWith($"squidex.teams.{TeamId}", StringComparison.OrdinalIgnoreCase))); Assert.True(isNextCalled); }