Browse Source

Feature/backup performance (#679)

* Code cleanup

* Batch store.

* Fixes.

* More performance improvements.

* Temp

* Fix parallelization.

* Ignore restored events for some consumers.

* Tests fixed.
pull/680/head
Sebastian Stehle 5 years ago
committed by GitHub
parent
commit
d124b1d428
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 7
      backend/src/Migrations/Migrations/ClearRules.cs
  2. 6
      backend/src/Migrations/Migrations/ClearSchemas.cs
  3. 6
      backend/src/Migrations/Migrations/CreateAssetSlugs.cs
  4. 2
      backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs
  5. 2
      backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs
  6. 49
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs
  7. 49
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs
  8. 10
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs
  9. 96
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs
  10. 5
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Schemas/MongoSchemasHash.cs
  11. 1
      backend/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs
  12. 4
      backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs
  13. 5
      backend/src/Squidex.Domain.Apps.Entities/Assets/AssetPermanentDeleter.cs
  14. 4
      backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs
  15. 4
      backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetFolderDomainObject.cs
  16. 5
      backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs
  17. 6
      backend/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs
  18. 5
      backend/src/Squidex.Domain.Apps.Entities/Backup/CompatibilityExtensions.cs
  19. 31
      backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreContext.cs
  20. 97
      backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs
  21. 76
      backend/src/Squidex.Domain.Apps.Entities/Backup/StreamMapper.cs
  22. 16
      backend/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs
  23. 4
      backend/src/Squidex.Domain.Apps.Entities/Contents/DomainObject/ContentDomainObject.cs
  24. 1
      backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/AllTypes.cs
  25. 2
      backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/Contents/ContentActions.cs
  26. 4
      backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs
  27. 4
      backend/src/Squidex.Domain.Apps.Entities/Rules/DomainObject/RuleDomainObject.cs
  28. 9
      backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs
  29. 4
      backend/src/Squidex.Domain.Apps.Entities/Schemas/DomainObject/SchemaDomainObject.cs
  30. 7
      backend/src/Squidex.Domain.Users/DefaultKeyStore.cs
  31. 6
      backend/src/Squidex.Domain.Users/DefaultXmlRepository.cs
  32. 29
      backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs
  33. 2
      backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs
  34. 10
      backend/src/Squidex.Infrastructure.MongoDb/Log/MongoRequestLogRepository.cs
  35. 2
      backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs
  36. 11
      backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoRepositoryBase.cs
  37. 37
      backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs
  38. 4
      backend/src/Squidex.Infrastructure.MongoDb/States/MongoState.cs
  39. 5
      backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs
  40. 14
      backend/src/Squidex.Infrastructure/Commands/DomainObject.cs
  41. 82
      backend/src/Squidex.Infrastructure/Commands/Rebuilder.cs
  42. 5
      backend/src/Squidex.Infrastructure/DomainId.cs
  43. 14
      backend/src/Squidex.Infrastructure/EventSourcing/CommonHeaders.cs
  44. 36
      backend/src/Squidex.Infrastructure/EventSourcing/EnvelopeExtensions.cs
  45. 4
      backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs
  46. 12
      backend/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs
  47. 4
      backend/src/Squidex.Infrastructure/Json/Newtonsoft/JsonValueConverter.cs
  48. 5
      backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs
  49. 14
      backend/src/Squidex.Infrastructure/Orleans/GrainState.cs
  50. 112
      backend/src/Squidex.Infrastructure/States/BatchContext.cs
  51. 80
      backend/src/Squidex.Infrastructure/States/BatchPersistence.cs
  52. 16
      backend/src/Squidex.Infrastructure/States/IBatchContext.cs
  53. 17
      backend/src/Squidex.Infrastructure/States/IPersistence.cs
  54. 25
      backend/src/Squidex.Infrastructure/States/IPersistenceFactory.cs
  55. 11
      backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs
  56. 16
      backend/src/Squidex.Infrastructure/States/IStore.cs
  57. 205
      backend/src/Squidex.Infrastructure/States/Persistence.cs
  58. 221
      backend/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs
  59. 59
      backend/src/Squidex.Infrastructure/States/Store.cs
  60. 17
      backend/src/Squidex.Infrastructure/States/StoreExtensions.cs
  61. 4
      backend/src/Squidex.Infrastructure/Tasks/PartitionedActionBlock.cs
  62. 10
      backend/src/Squidex/Config/Domain/StoreServices.cs
  63. 2
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs
  64. 11
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetPermanentDeleterTests.cs
  65. 2
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs
  66. 2
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetFolderDomainObjectTests.cs
  67. 11
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs
  68. 4
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupCompatibilityTests.cs
  69. 70
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/StreamMapperTests.cs
  70. 3
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/DomainObject/ContentDomainObjectTests.cs
  71. 2
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/DomainObject/RuleDomainObjectTests.cs
  72. 34
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs
  73. 2
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/DomainObject/SchemaDomainObjectTests.cs
  74. 21
      backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs
  75. 16
      backend/tests/Squidex.Domain.Users.Tests/DefaultKeyStoreTests.cs
  76. 5
      backend/tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs
  77. 20
      backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs
  78. 88
      backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs
  79. 164
      backend/tests/Squidex.Infrastructure.Tests/States/PersistenceBatchTests.cs
  80. 51
      backend/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs
  81. 39
      backend/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs
  82. 2
      backend/tests/Squidex.Infrastructure.Tests/Tasks/PartitionedActionBlockTests.cs
  83. 4
      backend/tests/Squidex.Infrastructure.Tests/TestHelpers/MyDomainObject.cs

7
backend/src/Migrations/Migrations/ClearRules.cs

@ -5,7 +5,6 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Rules.DomainObject;
using Squidex.Infrastructure.Migrations;
@ -15,16 +14,16 @@ namespace Migrations.Migrations
{
public sealed class ClearRules : IMigration
{
private readonly IStore<Guid> store;
private readonly IStore<RuleDomainObject.State> store;
public ClearRules(IStore<Guid> store)
public ClearRules(IStore<RuleDomainObject.State> store)
{
this.store = store;
}
public Task UpdateAsync()
{
return store.ClearSnapshotsAsync<Guid, RuleDomainObject.State>();
return store.ClearSnapshotsAsync();
}
}
}

6
backend/src/Migrations/Migrations/ClearSchemas.cs

@ -15,16 +15,16 @@ namespace Migrations.Migrations
{
public sealed class ClearSchemas : IMigration
{
private readonly IStore<Guid> store;
private readonly IStore<SchemaDomainObject.State> store;
public ClearSchemas(IStore<Guid> store)
public ClearSchemas(IStore<SchemaDomainObject.State> store)
{
this.store = store;
}
public Task UpdateAsync()
{
return store.ClearSnapshotsAsync<Guid, SchemaDomainObject.State>();
return store.ClearSnapshotsAsync();
}
}
}

6
backend/src/Migrations/Migrations/CreateAssetSlugs.cs

@ -16,9 +16,9 @@ namespace Migrations.Migrations
{
public sealed class CreateAssetSlugs : IMigration
{
private readonly ISnapshotStore<AssetDomainObject.State, string> stateForAssets;
private readonly ISnapshotStore<AssetDomainObject.State> stateForAssets;
public CreateAssetSlugs(ISnapshotStore<AssetDomainObject.State, string> stateForAssets)
public CreateAssetSlugs(ISnapshotStore<AssetDomainObject.State> stateForAssets)
{
this.stateForAssets = stateForAssets;
}
@ -29,7 +29,7 @@ namespace Migrations.Migrations
{
state.Slug = state.FileName.ToAssetSlug();
var key = DomainId.Combine(state.AppId.Id, state.Id).ToString();
var key = DomainId.Combine(state.AppId.Id, state.Id);
await stateForAssets.WriteAsync(key, state, version, version);
});

2
backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs

@ -105,7 +105,7 @@ namespace Migrations.Migrations.MongoDb
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 2,
MaxMessagesPerTask = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = SizeOfQueue
});

2
backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs

@ -139,7 +139,7 @@ namespace Migrations.Migrations.MongoDb
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 2,
MaxMessagesPerTask = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = SizeOfQueue
});

49
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs

@ -6,6 +6,8 @@
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
@ -19,9 +21,9 @@ using Squidex.Log;
namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
{
public sealed partial class MongoAssetFolderRepository : ISnapshotStore<AssetFolderDomainObject.State, DomainId>
public sealed partial class MongoAssetFolderRepository : ISnapshotStore<AssetFolderDomainObject.State>
{
async Task<(AssetFolderDomainObject.State Value, long Version)> ISnapshotStore<AssetFolderDomainObject.State, DomainId>.ReadAsync(DomainId key)
async Task<(AssetFolderDomainObject.State Value, long Version)> ISnapshotStore<AssetFolderDomainObject.State>.ReadAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoAssetFolderRepository>())
{
@ -38,19 +40,32 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
async Task ISnapshotStore<AssetFolderDomainObject.State, DomainId>.WriteAsync(DomainId key, AssetFolderDomainObject.State value, long oldVersion, long newVersion)
async Task ISnapshotStore<AssetFolderDomainObject.State>.WriteAsync(DomainId key, AssetFolderDomainObject.State value, long oldVersion, long newVersion)
{
using (Profiler.TraceMethod<MongoAssetFolderRepository>())
{
var entity = SimpleMapper.Map(value, new MongoAssetFolderEntity());
entity.IndexedAppId = value.AppId.Id;
var entity = Map(value);
await Collection.UpsertVersionedAsync(key, oldVersion, newVersion, entity);
}
}
async Task ISnapshotStore<AssetFolderDomainObject.State, DomainId>.ReadAllAsync(Func<AssetFolderDomainObject.State, long, Task> callback, CancellationToken ct)
async Task ISnapshotStore<AssetFolderDomainObject.State>.WriteManyAsync(IEnumerable<(DomainId Key, AssetFolderDomainObject.State Value, long Version)> snapshots)
{
using (Profiler.TraceMethod<MongoAssetFolderRepository>())
{
var entities = snapshots.Select(Map).ToList();
if (entities.Count == 0)
{
return;
}
await Collection.InsertManyAsync(entities, InsertUnordered);
}
}
async Task ISnapshotStore<AssetFolderDomainObject.State>.ReadAllAsync(Func<AssetFolderDomainObject.State, long, Task> callback, CancellationToken ct)
{
using (Profiler.TraceMethod<MongoAssetFolderRepository>())
{
@ -58,7 +73,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
async Task ISnapshotStore<AssetFolderDomainObject.State, DomainId>.RemoveAsync(DomainId key)
async Task ISnapshotStore<AssetFolderDomainObject.State>.RemoveAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoAssetFolderRepository>())
{
@ -66,6 +81,24 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
private static MongoAssetFolderEntity Map(AssetFolderDomainObject.State value)
{
var entity = SimpleMapper.Map(value, new MongoAssetFolderEntity());
entity.IndexedAppId = value.AppId.Id;
return entity;
}
private static MongoAssetFolderEntity Map((DomainId Key, AssetFolderDomainObject.State Value, long Version) snapshot)
{
var entity = Map(snapshot.Value);
entity.DocumentId = snapshot.Key;
return entity;
}
private static AssetFolderDomainObject.State Map(MongoAssetFolderEntity existing)
{
return SimpleMapper.Map(existing, new AssetFolderDomainObject.State());

49
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs

@ -6,6 +6,8 @@
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
@ -19,9 +21,9 @@ using Squidex.Log;
namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
{
public sealed partial class MongoAssetRepository : ISnapshotStore<AssetDomainObject.State, DomainId>
public sealed partial class MongoAssetRepository : ISnapshotStore<AssetDomainObject.State>
{
async Task<(AssetDomainObject.State Value, long Version)> ISnapshotStore<AssetDomainObject.State, DomainId>.ReadAsync(DomainId key)
async Task<(AssetDomainObject.State Value, long Version)> ISnapshotStore<AssetDomainObject.State>.ReadAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
{
@ -38,19 +40,32 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
async Task ISnapshotStore<AssetDomainObject.State, DomainId>.WriteAsync(DomainId key, AssetDomainObject.State value, long oldVersion, long newVersion)
async Task ISnapshotStore<AssetDomainObject.State>.WriteAsync(DomainId key, AssetDomainObject.State value, long oldVersion, long newVersion)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
{
var entity = SimpleMapper.Map(value, new MongoAssetEntity());
entity.IndexedAppId = value.AppId.Id;
var entity = Map(value);
await Collection.UpsertVersionedAsync(key, oldVersion, newVersion, entity);
}
}
async Task ISnapshotStore<AssetDomainObject.State, DomainId>.ReadAllAsync(Func<AssetDomainObject.State, long, Task> callback, CancellationToken ct)
async Task ISnapshotStore<AssetDomainObject.State>.WriteManyAsync(IEnumerable<(DomainId Key, AssetDomainObject.State Value, long Version)> snapshots)
{
using (Profiler.TraceMethod<MongoAssetFolderRepository>())
{
var entities = snapshots.Select(Map).ToList();
if (entities.Count == 0)
{
return;
}
await Collection.InsertManyAsync(entities, InsertUnordered);
}
}
async Task ISnapshotStore<AssetDomainObject.State>.ReadAllAsync(Func<AssetDomainObject.State, long, Task> callback, CancellationToken ct)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
{
@ -58,7 +73,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
async Task ISnapshotStore<AssetDomainObject.State, DomainId>.RemoveAsync(DomainId key)
async Task ISnapshotStore<AssetDomainObject.State>.RemoveAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
{
@ -66,6 +81,24 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
private static MongoAssetEntity Map(AssetDomainObject.State value)
{
var entity = SimpleMapper.Map(value, new MongoAssetEntity());
entity.IndexedAppId = value.AppId.Id;
return entity;
}
private static MongoAssetEntity Map((DomainId Key, AssetDomainObject.State Value, long Version) snapshot)
{
var entity = Map(snapshot.Value);
entity.DocumentId = snapshot.Key;
return entity;
}
private static AssetDomainObject.State Map(MongoAssetEntity existing)
{
return SimpleMapper.Map(existing, new AssetDomainObject.State());

10
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs

@ -187,5 +187,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
{
return Collection.DeleteOneAsync(x => x.DocumentId == documentId);
}
public Task InsertManyAsync(IReadOnlyList<MongoContentEntity> entities)
{
if (entities.Count == 0)
{
return Task.CompletedTask;
}
return Collection.InsertManyAsync(entities, InsertUnordered);
}
}
}

96
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs

@ -19,19 +19,19 @@ using Squidex.Log;
namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
{
public partial class MongoContentRepository : ISnapshotStore<ContentDomainObject.State, DomainId>
public partial class MongoContentRepository : ISnapshotStore<ContentDomainObject.State>
{
Task ISnapshotStore<ContentDomainObject.State, DomainId>.ReadAllAsync(Func<ContentDomainObject.State, long, Task> callback, CancellationToken ct)
Task ISnapshotStore<ContentDomainObject.State>.ReadAllAsync(Func<ContentDomainObject.State, long, Task> callback, CancellationToken ct)
{
throw new NotSupportedException();
}
Task<(ContentDomainObject.State Value, long Version)> ISnapshotStore<ContentDomainObject.State, DomainId>.ReadAsync(DomainId key)
Task<(ContentDomainObject.State Value, long Version)> ISnapshotStore<ContentDomainObject.State>.ReadAsync(DomainId key)
{
return Task.FromResult<(ContentDomainObject.State, long Version)>((null!, EtagVersion.Empty));
}
async Task ISnapshotStore<ContentDomainObject.State, DomainId>.ClearAsync()
async Task ISnapshotStore<ContentDomainObject.State>.ClearAsync()
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
@ -40,7 +40,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
}
}
async Task ISnapshotStore<ContentDomainObject.State, DomainId>.RemoveAsync(DomainId key)
async Task ISnapshotStore<ContentDomainObject.State>.RemoveAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
@ -49,7 +49,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
}
}
async Task ISnapshotStore<ContentDomainObject.State, DomainId>.WriteAsync(DomainId key, ContentDomainObject.State value, long oldVersion, long newVersion)
async Task ISnapshotStore<ContentDomainObject.State>.WriteAsync(DomainId key, ContentDomainObject.State value, long oldVersion, long newVersion)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
@ -64,9 +64,32 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
}
}
async Task ISnapshotStore<ContentDomainObject.State>.WriteManyAsync(IEnumerable<(DomainId Key, ContentDomainObject.State Value, long Version)> snapshots)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
var entitiesPublished = new List<MongoContentEntity>();
var entitiesAll = new List<MongoContentEntity>();
foreach (var (_, value, version) in snapshots)
{
if (ShouldWritePublished(value))
{
entitiesPublished.Add(await CreatePublishedContentAsync(value, version));
}
entitiesAll.Add(await CreateDraftContentAsync(value, version));
}
await Task.WhenAll(
collectionPublished.InsertManyAsync(entitiesPublished),
collectionAll.InsertManyAsync(entitiesAll));
}
}
private async Task UpsertOrDeletePublishedAsync(ContentDomainObject.State value, long oldVersion, long newVersion)
{
if (value.Status == Status.Published && !value.IsDeleted)
if (ShouldWritePublished(value))
{
await UpsertPublishedContentAsync(value, oldVersion, newVersion);
}
@ -85,48 +108,67 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
private async Task UpsertDraftContentAsync(ContentDomainObject.State value, long oldVersion, long newVersion)
{
var content = await CreateContentAsync(value, value.Data, newVersion);
content.ScheduledAt = value.ScheduleJob?.DueTime;
content.ScheduleJob = value.ScheduleJob;
content.NewStatus = value.NewStatus;
var entity = await CreateDraftContentAsync(value, newVersion);
await collectionAll.UpsertVersionedAsync(content.DocumentId, oldVersion, content);
await collectionAll.UpsertVersionedAsync(entity.DocumentId, oldVersion, entity);
}
private async Task UpsertPublishedContentAsync(ContentDomainObject.State value, long oldVersion, long newVersion)
{
var content = await CreateContentAsync(value, value.CurrentVersion.Data, newVersion);
var entity = await CreatePublishedContentAsync(value, newVersion);
content.ScheduledAt = null;
content.ScheduleJob = null;
content.NewStatus = null;
await collectionPublished.UpsertVersionedAsync(entity.DocumentId, oldVersion, entity);
}
private async Task<MongoContentEntity> CreatePublishedContentAsync(ContentDomainObject.State value, long newVersion)
{
var entity = await CreateContentAsync(value, value.CurrentVersion.Data, newVersion);
entity.ScheduledAt = null;
entity.ScheduleJob = null;
entity.NewStatus = null;
return entity;
}
await collectionPublished.UpsertVersionedAsync(content.DocumentId, oldVersion, content);
private async Task<MongoContentEntity> CreateDraftContentAsync(ContentDomainObject.State value, long newVersion)
{
var entity = await CreateContentAsync(value, value.Data, newVersion);
entity.ScheduledAt = value.ScheduleJob?.DueTime;
entity.ScheduleJob = value.ScheduleJob;
entity.NewStatus = value.NewStatus;
return entity;
}
private async Task<MongoContentEntity> CreateContentAsync(ContentDomainObject.State value, ContentData data, long newVersion)
{
var content = SimpleMapper.Map(value, new MongoContentEntity());
var entity = SimpleMapper.Map(value, new MongoContentEntity());
content.Data = data;
content.DocumentId = value.UniqueId;
content.IndexedAppId = value.AppId.Id;
content.IndexedSchemaId = value.SchemaId.Id;
content.Version = newVersion;
entity.Data = data;
entity.DocumentId = value.UniqueId;
entity.IndexedAppId = value.AppId.Id;
entity.IndexedSchemaId = value.SchemaId.Id;
entity.Version = newVersion;
var schema = await appProvider.GetSchemaAsync(value.AppId.Id, value.SchemaId.Id, true);
if (schema != null)
{
content.ReferencedIds = content.Data.GetReferencedIds(schema.SchemaDef);
entity.ReferencedIds = entity.Data.GetReferencedIds(schema.SchemaDef);
}
else
{
content.ReferencedIds = new HashSet<DomainId>();
entity.ReferencedIds = new HashSet<DomainId>();
}
return content;
return entity;
}
private static bool ShouldWritePublished(ContentDomainObject.State value)
{
return value.Status == Status.Published && !value.IsDeleted;
}
}
}

5
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Schemas/MongoSchemasHash.cs

@ -58,6 +58,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Schemas
foreach (var @event in events)
{
if (@event.Headers.Restored())
{
continue;
}
switch (@event.Payload)
{
case SchemaEvent schemaEvent:

1
backend/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs

@ -135,7 +135,6 @@ namespace Squidex.Domain.Apps.Entities.Apps
public async Task CompleteRestoreAsync(RestoreContext context)
{
await appsIndex.AddAsync(appReservation);
await appsIndex.RebuildByContributorsAsync(context.AppId, contributors);
}

4
backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs

@ -31,12 +31,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
private readonly IAppPlanBillingManager appPlansBillingManager;
private readonly IUserResolver userResolver;
public AppDomainObject(IStore<DomainId> store, ISemanticLog log,
public AppDomainObject(IPersistenceFactory<State> persistence, ISemanticLog log,
InitialPatterns initialPatterns,
IAppPlansProvider appPlansProvider,
IAppPlanBillingManager appPlansBillingManager,
IUserResolver userResolver)
: base(store, log)
: base(persistence, log)
{
Guard.NotNull(initialPatterns, nameof(initialPatterns));
Guard.NotNull(userResolver, nameof(userResolver));

5
backend/src/Squidex.Domain.Apps.Entities/Assets/AssetPermanentDeleter.cs

@ -45,6 +45,11 @@ namespace Squidex.Domain.Apps.Entities.Assets
public async Task On(Envelope<IEvent> @event)
{
if (@event.Headers.Restored())
{
return;
}
if (@event.Payload is AssetDeleted assetDeleted)
{
for (var version = 0; version < @event.Headers.EventStreamNumber(); version++)

4
backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs

@ -30,11 +30,11 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
private readonly IAssetTagService assetTags;
private readonly IAssetQueryService assetQuery;
public AssetDomainObject(IStore<DomainId> store, ISemanticLog log,
public AssetDomainObject(IPersistenceFactory<AssetDomainObject.State> factory, ISemanticLog log,
IAssetTagService assetTags,
IAssetQueryService assetQuery,
IContentRepository contentRepository)
: base(store, log)
: base(factory, log)
{
Guard.NotNull(assetTags, nameof(assetTags));
Guard.NotNull(assetQuery, nameof(assetQuery));

4
backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetFolderDomainObject.cs

@ -24,9 +24,9 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
{
private readonly IAssetQueryService assetQuery;
public AssetFolderDomainObject(IStore<DomainId> store, ISemanticLog log,
public AssetFolderDomainObject(IPersistenceFactory<State> factory, ISemanticLog log,
IAssetQueryService assetQuery)
: base(store, log)
: base(factory, log)
{
Guard.NotNull(assetQuery, nameof(assetQuery));

5
backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs

@ -63,6 +63,11 @@ namespace Squidex.Domain.Apps.Entities.Assets
public async Task On(Envelope<IEvent> @event)
{
if (@event.Headers.Restored())
{
return;
}
if (@event.Payload is AssetFolderDeleted folderDeleted)
{
async Task PublishAsync(SquidexCommand command)

6
backend/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs

@ -99,14 +99,14 @@ namespace Squidex.Domain.Apps.Entities.Backup
while (true)
{
var eventEntry = archive.GetEntry(ArchiveHelper.GetEventPath(readEvents));
var entry = archive.GetEntry(ArchiveHelper.GetEventPath(readEvents));
if (eventEntry == null)
if (entry == null)
{
break;
}
using (var stream = eventEntry.Open())
using (var stream = entry.Open())
{
var storedEvent = serializer.Deserialize<CompatibleStoredEvent>(stream).ToStoredEvent();

5
backend/src/Squidex.Domain.Apps.Entities/Backup/CompatibilityExtensions.cs

@ -34,6 +34,11 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
var current = await reader.ReadVersionAsync();
if (None.Equals(current))
{
return;
}
if (!Expected.Equals(current))
{
throw new BackupRestoreException("Backup file is not compatible with this version.");

31
backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreContext.cs

@ -5,17 +5,12 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using Squidex.Infrastructure;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class RestoreContext : BackupContextBase
{
private readonly Dictionary<string, long> streams = new Dictionary<string, long>(1000);
private string? appStream;
public IBackupReader Reader { get; }
public DomainId PreviousAppId { get; set; }
@ -29,31 +24,5 @@ namespace Squidex.Domain.Apps.Entities.Backup
PreviousAppId = previousAppId;
}
public string GetStreamName(string streamName)
{
Guard.NotNullOrEmpty(streamName, nameof(streamName));
if (streamName.StartsWith("app-", StringComparison.OrdinalIgnoreCase))
{
return appStream ??= $"app-{AppId}";
}
return streamName.Replace(PreviousAppId.ToString(), AppId.ToString());
}
public long GetStreamOffset(string streamName)
{
Guard.NotNullOrEmpty(streamName, nameof(streamName));
if (!streams.TryGetValue(streamName, out var offset))
{
offset = EtagVersion.Empty;
}
streams[streamName] = offset + 1;
return offset;
}
}
}

97
backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs

@ -8,7 +8,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Microsoft.Extensions.DependencyInjection;
using NodaTime;
using Squidex.Domain.Apps.Core.Apps;
@ -19,7 +21,6 @@ using Squidex.Domain.Apps.Events.Apps;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Json.Objects;
using Squidex.Infrastructure.Orleans;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
@ -31,7 +32,6 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class RestoreGrain : GrainOfString, IRestoreGrain
{
private const int BatchSize = 500;
private readonly IBackupArchiveLocation backupArchiveLocation;
private readonly IClock clock;
private readonly ICommandBus commandBus;
@ -42,7 +42,8 @@ namespace Squidex.Domain.Apps.Entities.Backup
private readonly IStreamNameResolver streamNameResolver;
private readonly IUserResolver userResolver;
private readonly IGrainState<BackupRestoreState> state;
private RestoreContext restoreContext;
private RestoreContext runningContext;
private StreamMapper runningStreamMapper;
private RestoreJob CurrentJob
{
@ -176,7 +177,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
using (Profiler.TraceMethod(handler.GetType(), nameof(IBackupHandler.RestoreAsync)))
{
await handler.RestoreAsync(restoreContext);
await handler.RestoreAsync(runningContext);
}
Log($"Restored {handler.Name}");
@ -186,7 +187,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
using (Profiler.TraceMethod(handler.GetType(), nameof(IBackupHandler.CompleteRestoreAsync)))
{
await handler.CompleteRestoreAsync(restoreContext);
await handler.CompleteRestoreAsync(runningContext);
}
Log($"Completed {handler.Name}");
@ -243,6 +244,9 @@ namespace Squidex.Domain.Apps.Entities.Backup
CurrentJob.Stopped = clock.GetCurrentInstant();
await state.WriteAsync();
runningStreamMapper = null!;
runningContext = null!;
}
}
}
@ -316,51 +320,59 @@ namespace Squidex.Domain.Apps.Entities.Backup
private async Task ReadEventsAsync(IBackupReader reader, IEnumerable<IBackupHandler> handlers)
{
var batch = new List<(string, Envelope<IEvent>)>(BatchSize);
const int BatchSize = 500;
var handled = 0;
await reader.ReadEventsAsync(streamNameResolver, eventDataFormatter, async storedEvent =>
var writeBlock = new ActionBlock<(string, Envelope<IEvent>)[]>(async batch =>
{
batch.Add(storedEvent);
var commits = new List<EventCommit>(batch.Length);
if (batch.Count == BatchSize)
foreach (var (stream, @event) in batch)
{
await CommitBatchAsync(reader, handlers, batch);
var offset = runningStreamMapper.GetStreamOffset(stream);
batch.Clear();
commits.Add(EventCommit.Create(stream, offset, @event, eventDataFormatter));
}
});
if (batch.Count > 0)
await eventStore.AppendUnsafeAsync(commits);
handled += commits.Count;
Log($"Reading {reader.ReadEvents}/{handled} events and {reader.ReadAttachments} attachments completed.", true);
}, new ExecutionDataflowBlockOptions
{
await CommitBatchAsync(reader, handlers, batch);
}
MaxDegreeOfParallelism = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = 2
});
Log($"Reading {reader.ReadEvents} events and {reader.ReadAttachments} attachments completed.", true);
}
var batchBlock = new BatchBlock<(string, Envelope<IEvent>)>(500, new GroupingDataflowBlockOptions
{
BoundedCapacity = BatchSize
});
private async Task CommitBatchAsync(IBackupReader reader, IEnumerable<IBackupHandler> handlers, List<(string, Envelope<IEvent>)> batch)
{
var commits = new List<EventCommit>(batch.Count);
batchBlock.LinkTo(writeBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
foreach (var (stream, @event) in batch)
await reader.ReadEventsAsync(streamNameResolver, eventDataFormatter, async job =>
{
var handled = await HandleEventAsync(reader, handlers, stream, @event);
var newStream = await HandleEventAsync(reader, handlers, job.Stream, job.Event);
if (handled)
if (newStream != null)
{
var streamName = restoreContext.GetStreamName(stream);
var streamOffset = restoreContext.GetStreamOffset(streamName);
commits.Add(EventCommit.Create(streamName, streamOffset, @event, eventDataFormatter));
await batchBlock.SendAsync((newStream, job.Event));
}
}
});
await eventStore.AppendUnsafeAsync(commits);
batchBlock.Complete();
Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments.", true);
await writeBlock.Completion;
}
private async Task<bool> HandleEventAsync(IBackupReader reader, IEnumerable<IBackupHandler> handlers, string stream, Envelope<IEvent> @event)
private async Task<string?> HandleEventAsync(IBackupReader reader, IEnumerable<IBackupHandler> handlers, string stream, Envelope<IEvent> @event)
{
if (@event.Payload is AppCreated appCreated)
{
@ -382,7 +394,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
if (@event.Payload is SquidexEvent squidexEvent && squidexEvent.Actor != null)
{
if (restoreContext.UserMapping.TryMap(squidexEvent.Actor, out var newUser))
if (runningContext.UserMapping.TryMap(squidexEvent.Actor, out var newUser))
{
squidexEvent.Actor = newUser;
}
@ -393,26 +405,20 @@ namespace Squidex.Domain.Apps.Entities.Backup
appEvent.AppId = CurrentJob.AppId;
}
if (@event.Headers.TryGet(CommonHeaders.AggregateId, out var value) && value is JsonString idString)
{
var id = idString.Value.Replace(
restoreContext.PreviousAppId.ToString(),
restoreContext.AppId.ToString());
var domainId = DomainId.Create(id);
var (newStream, id) = runningStreamMapper.Map(stream);
@event.SetAggregateId(domainId);
}
@event.SetAggregateId(id);
@event.SetRestored();
foreach (var handler in handlers)
{
if (!await handler.RestoreEventAsync(@event, restoreContext))
if (!await handler.RestoreEventAsync(@event, runningContext))
{
return false;
return null;
}
}
return true;
return newStream;
}
private async Task CreateContextAsync(IBackupReader reader, DomainId previousAppId)
@ -428,7 +434,8 @@ namespace Squidex.Domain.Apps.Entities.Backup
Log("Created Users");
}
restoreContext = new RestoreContext(CurrentJob.AppId.Id, userMapping, reader, previousAppId);
runningContext = new RestoreContext(CurrentJob.AppId.Id, userMapping, reader, previousAppId);
runningStreamMapper = new StreamMapper(runningContext);
}
private void Log(string message, bool replace = false)

76
backend/src/Squidex.Domain.Apps.Entities/Backup/StreamMapper.cs

@ -0,0 +1,76 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using Squidex.Infrastructure;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class StreamMapper
{
private readonly Dictionary<string, long> streams = new Dictionary<string, long>(1000);
private readonly RestoreContext context;
private readonly DomainId brokenAppId;
public StreamMapper(RestoreContext context)
{
Guard.NotNull(context, nameof(context));
this.context = context;
brokenAppId = DomainId.Combine(context.PreviousAppId, context.PreviousAppId);
}
public (string Stream, DomainId) Map(string stream)
{
Guard.NotNullOrEmpty(stream, nameof(stream));
var typeIndex = stream.IndexOf("-", StringComparison.Ordinal);
var typeName = stream.Substring(0, typeIndex);
var id = DomainId.Create(stream[(typeIndex + 1)..]);
if (id.Equals(context.PreviousAppId) || id.Equals(brokenAppId))
{
id = context.AppId;
}
else
{
var separator = DomainId.IdSeparator;
var secondId = id.ToString().AsSpan();
var indexOfSecondPart = secondId.IndexOf(separator, StringComparison.Ordinal);
if (indexOfSecondPart > 0 && indexOfSecondPart < secondId.Length - separator.Length - 1)
{
secondId = secondId[(indexOfSecondPart + separator.Length)..];
}
id = DomainId.Combine(context.AppId, DomainId.Create(secondId.ToString()));
}
stream = $"{typeName}-{id}";
return (stream, id);
}
public long GetStreamOffset(string streamName)
{
Guard.NotNullOrEmpty(streamName, nameof(streamName));
if (!streams.TryGetValue(streamName, out var offset))
{
offset = EtagVersion.Empty;
}
streams[streamName] = offset + 1;
return offset;
}
}
}

16
backend/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs

@ -29,24 +29,16 @@ namespace Squidex.Domain.Apps.Entities.Backup
public async Task<IBackupReader> OpenReaderAsync(Uri url, DomainId id)
{
var stream = OpenStream(id);
Stream stream;
if (string.Equals(url.Scheme, "file"))
{
try
{
using (var sourceStream = new FileStream(url.LocalPath, FileMode.Open, FileAccess.Read))
{
await sourceStream.CopyToAsync(stream);
}
}
catch (IOException ex)
{
throw new BackupRestoreException($"Cannot download the archive: {ex.Message}.", ex);
}
stream = new FileStream(url.LocalPath, FileMode.Open, FileAccess.Read);
}
else
{
stream = OpenStream(id);
HttpResponseMessage? response = null;
try
{

4
backend/src/Squidex.Domain.Apps.Entities/Contents/DomainObject/ContentDomainObject.cs

@ -28,9 +28,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.DomainObject
{
private readonly IServiceProvider serviceProvider;
public ContentDomainObject(IStore<DomainId> store, ISemanticLog log,
public ContentDomainObject(IPersistenceFactory<State> persistence, ISemanticLog log,
IServiceProvider serviceProvider)
: base(store, log)
: base(persistence, log)
{
Guard.NotNull(serviceProvider, nameof(serviceProvider));

1
backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/AllTypes.cs

@ -5,7 +5,6 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using GraphQL.Types;
using Squidex.Domain.Apps.Core.Assets;
using Squidex.Domain.Apps.Entities.Contents.GraphQL.Types.Primitives;

2
backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/Contents/ContentActions.cs

@ -6,14 +6,12 @@
// ==========================================================================
using System;
using System.Collections.Generic;
using GraphQL;
using GraphQL.Resolvers;
using GraphQL.Types;
using NodaTime;
using Squidex.Domain.Apps.Core.Contents;
using Squidex.Domain.Apps.Entities.Contents.Commands;
using Squidex.Domain.Apps.Entities.Contents.GraphQL.Types.Primitives;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Json.Objects;
using Squidex.Infrastructure.Translations;

4
backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs

@ -127,9 +127,9 @@ namespace Squidex.Domain.Apps.Entities.History
var now = clock.GetCurrentInstant();
var publishedEvents = events
.Where(x => x.AppEvent.Headers.Restored() == false)
.Where(x => IsTooOld(x.AppEvent.Headers, now) == false)
.Where(x => IsComment(x.AppEvent.Payload) || x.HistoryEvent != null)
.ToList();
.Where(x => IsComment(x.AppEvent.Payload) || x.HistoryEvent != null);
foreach (var batch in publishedEvents.Batch(50))
{

4
backend/src/Squidex.Domain.Apps.Entities/Rules/DomainObject/RuleDomainObject.cs

@ -25,9 +25,9 @@ namespace Squidex.Domain.Apps.Entities.Rules.DomainObject
private readonly IAppProvider appProvider;
private readonly IRuleEnqueuer ruleEnqueuer;
public RuleDomainObject(IStore<DomainId> store, ISemanticLog log,
public RuleDomainObject(IPersistenceFactory<State> factory, ISemanticLog log,
IAppProvider appProvider, IRuleEnqueuer ruleEnqueuer)
: base(store, log)
: base(factory, log)
{
Guard.NotNull(appProvider, nameof(appProvider));
Guard.NotNull(ruleEnqueuer, nameof(ruleEnqueuer));

9
backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs

@ -66,9 +66,14 @@ namespace Squidex.Domain.Apps.Entities.Rules
public async Task On(Envelope<IEvent> @event)
{
using (localCache.StartContext())
if (@event.Headers.Restored())
{
if (@event.Payload is AppEvent appEvent)
return;
}
if (@event.Payload is AppEvent appEvent)
{
using (localCache.StartContext())
{
var rules = await GetRulesAsync(appEvent.AppId.Id);

4
backend/src/Squidex.Domain.Apps.Entities/Schemas/DomainObject/SchemaDomainObject.cs

@ -25,8 +25,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas.DomainObject
{
public sealed partial class SchemaDomainObject : DomainObject<SchemaDomainObject.State>
{
public SchemaDomainObject(IStore<DomainId> store, ISemanticLog log)
: base(store, log)
public SchemaDomainObject(IPersistenceFactory<State> persistence, ISemanticLog log)
: base(persistence, log)
{
}

7
backend/src/Squidex.Domain.Users/DefaultKeyStore.cs

@ -13,13 +13,14 @@ using IdentityModel;
using IdentityServer4.Models;
using IdentityServer4.Stores;
using Microsoft.IdentityModel.Tokens;
using Squidex.Infrastructure;
using Squidex.Infrastructure.States;
namespace Squidex.Domain.Users
{
public sealed class DefaultKeyStore : ISigningCredentialStore, IValidationKeysStore
{
private readonly ISnapshotStore<State, Guid> store;
private readonly ISnapshotStore<State> store;
private SigningCredentials? cachedKey;
private SecurityKeyInfo[]? cachedKeyInfo;
@ -31,8 +32,10 @@ namespace Squidex.Domain.Users
public RSAParameters Parameters { get; set; }
}
public DefaultKeyStore(ISnapshotStore<State, Guid> store)
public DefaultKeyStore(ISnapshotStore<State> store)
{
Guard.NotNull(store, nameof(store));
this.store = store;
}

6
backend/src/Squidex.Domain.Users/DefaultXmlRepository.cs

@ -16,7 +16,7 @@ namespace Squidex.Domain.Users
{
public sealed class DefaultXmlRepository : IXmlRepository
{
private readonly ISnapshotStore<State, string> store;
private readonly ISnapshotStore<State> store;
[CollectionName("Identity_Xml")]
public sealed class State
@ -38,7 +38,7 @@ namespace Squidex.Domain.Users
}
}
public DefaultXmlRepository(ISnapshotStore<State, string> store)
public DefaultXmlRepository(ISnapshotStore<State> store)
{
Guard.NotNull(store, nameof(store));
@ -63,7 +63,7 @@ namespace Squidex.Domain.Users
{
var state = new State(element);
store.WriteAsync(friendlyName, state, EtagVersion.Any, 0);
store.WriteAsync(DomainId.Create(friendlyName), state, EtagVersion.Any, 0);
}
}
}

29
backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs

@ -95,12 +95,31 @@ namespace Squidex.Infrastructure.EventSourcing
Filter.Gte(EventStreamOffsetField, streamPosition - MaxCommitSize)))
.Sort(Sort.Ascending(TimestampField)).ToListAsync();
var result = new List<StoredEvent>();
var result = commits.SelectMany(x => x.Filtered(streamPosition)).ToList();
foreach (var commit in commits)
{
result.AddRange(commit.Filtered(streamPosition));
}
return result;
}
}
public async Task<IReadOnlyDictionary<string, IReadOnlyList<StoredEvent>>> QueryManyAsync(IEnumerable<string> streamNames)
{
Guard.NotNull(streamNames, nameof(streamNames));
using (Profiler.TraceMethod<MongoEventStore>())
{
var position = EtagVersion.Empty;
var commits =
await Collection.Find(
Filter.And(
Filter.In(EventStreamField, streamNames),
Filter.Gte(EventStreamOffsetField, position)))
.Sort(Sort.Ascending(TimestampField)).ToListAsync();
var result = commits.GroupBy(x => x.EventStream)
.ToDictionary(
x => x.Key,
x => (IReadOnlyList<StoredEvent>)x.SelectMany(y => y.Filtered(position)).ToList());
return result;
}

2
backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs

@ -115,7 +115,7 @@ namespace Squidex.Infrastructure.EventSourcing
if (writes.Count > 0)
{
await Collection.BulkWriteAsync(writes, Unordered);
await Collection.BulkWriteAsync(writes, BulkUnordered);
}
}
}

10
backend/src/Squidex.Infrastructure.MongoDb/Log/MongoRequestLogRepository.cs

@ -19,7 +19,6 @@ namespace Squidex.Infrastructure.Log
{
public sealed class MongoRequestLogRepository : MongoRepositoryBase<MongoRequest>, IRequestLogRepository
{
private static readonly InsertManyOptions Unordered = new InsertManyOptions { IsOrdered = false };
private readonly RequestLogStoreOptions options;
public MongoRequestLogRepository(IMongoDatabase database, IOptions<RequestLogStoreOptions> options)
@ -57,9 +56,14 @@ namespace Squidex.Infrastructure.Log
{
Guard.NotNull(items, nameof(items));
var documents = items.Select(x => new MongoRequest { Key = x.Key, Timestamp = x.Timestamp, Properties = x.Properties });
var entities = items.Select(x => new MongoRequest { Key = x.Key, Timestamp = x.Timestamp, Properties = x.Properties }).ToList();
return Collection.InsertManyAsync(documents, Unordered);
if (entities.Count == 0)
{
return Task.CompletedTask;
}
return Collection.InsertManyAsync(entities, InsertUnordered);
}
public Task QueryAllAsync(Func<Request, Task> callback, string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default)

2
backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs

@ -198,7 +198,7 @@ namespace Squidex.Infrastructure.MongoDb
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
MaxMessagesPerTask = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = Batching.BufferSize
});
try

11
backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoRepositoryBase.cs

@ -21,15 +21,16 @@ namespace Squidex.Infrastructure.MongoDb
{
private const string CollectionFormat = "{0}Set";
protected static readonly BulkWriteOptions Unordered = new BulkWriteOptions { IsOrdered = true };
protected static readonly UpdateOptions Upsert = new UpdateOptions { IsUpsert = true };
protected static readonly ReplaceOptions UpsertReplace = new ReplaceOptions { IsUpsert = true };
protected static readonly SortDefinitionBuilder<TEntity> Sort = Builders<TEntity>.Sort;
protected static readonly UpdateDefinitionBuilder<TEntity> Update = Builders<TEntity>.Update;
protected static readonly BulkWriteOptions BulkUnordered = new BulkWriteOptions { IsOrdered = true };
protected static readonly FieldDefinitionBuilder<TEntity> FieldBuilder = FieldDefinitionBuilder<TEntity>.Instance;
protected static readonly FilterDefinitionBuilder<TEntity> Filter = Builders<TEntity>.Filter;
protected static readonly IndexKeysDefinitionBuilder<TEntity> Index = Builders<TEntity>.IndexKeys;
protected static readonly InsertManyOptions InsertUnordered = new InsertManyOptions { IsOrdered = true };
protected static readonly ProjectionDefinitionBuilder<TEntity> Projection = Builders<TEntity>.Projection;
protected static readonly ReplaceOptions UpsertReplace = new ReplaceOptions { IsUpsert = true };
protected static readonly SortDefinitionBuilder<TEntity> Sort = Builders<TEntity>.Sort;
protected static readonly UpdateDefinitionBuilder<TEntity> Update = Builders<TEntity>.Update;
protected static readonly UpdateOptions Upsert = new UpdateOptions { IsUpsert = true };
private readonly IMongoDatabase mongoDatabase;
private IMongoCollection<TEntity> mongoCollection;

37
backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs

@ -6,6 +6,7 @@
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@ -17,7 +18,7 @@ using Squidex.Log;
namespace Squidex.Infrastructure.States
{
public class MongoSnapshotStore<T, TKey> : MongoRepositoryBase<MongoState<T, TKey>>, ISnapshotStore<T, TKey> where TKey : notnull
public class MongoSnapshotStore<T> : MongoRepositoryBase<MongoState<T>>, ISnapshotStore<T>
{
public MongoSnapshotStore(IMongoDatabase database, JsonSerializer jsonSerializer)
: base(database, Register(jsonSerializer))
@ -42,9 +43,9 @@ namespace Squidex.Infrastructure.States
return $"States_{name}";
}
public async Task<(T Value, long Version)> ReadAsync(TKey key)
public async Task<(T Value, long Version)> ReadAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
using (Profiler.TraceMethod<MongoSnapshotStore<T>>())
{
var existing =
await Collection.Find(x => x.DocumentId.Equals(key))
@ -59,25 +60,45 @@ namespace Squidex.Infrastructure.States
}
}
public async Task WriteAsync(TKey key, T value, long oldVersion, long newVersion)
public async Task WriteAsync(DomainId key, T value, long oldVersion, long newVersion)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
using (Profiler.TraceMethod<MongoSnapshotStore<T>>())
{
await Collection.UpsertVersionedAsync(key, oldVersion, newVersion, u => u.Set(x => x.Doc, value));
}
}
public Task WriteManyAsync(IEnumerable<(DomainId Key, T Value, long Version)> snapshots)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T>>())
{
var writes = snapshots.Select(x => new InsertOneModel<MongoState<T>>(new MongoState<T>
{
Doc = x.Value,
DocumentId = x.Key,
Version = x.Version
})).ToList();
if (writes.Count == 0)
{
return Task.CompletedTask;
}
return Collection.BulkWriteAsync(writes, BulkUnordered);
}
}
public async Task ReadAllAsync(Func<T, long, Task> callback, CancellationToken ct = default)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
using (Profiler.TraceMethod<MongoSnapshotStore<T>>())
{
await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(x.Doc, x.Version), ct);
}
}
public async Task RemoveAsync(TKey key)
public async Task RemoveAsync(DomainId key)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
using (Profiler.TraceMethod<MongoSnapshotStore<T>>())
{
await Collection.DeleteOneAsync(x => x.DocumentId.Equals(key));
}

4
backend/src/Squidex.Infrastructure.MongoDb/States/MongoState.cs

@ -12,12 +12,12 @@ using Squidex.Infrastructure.MongoDb;
namespace Squidex.Infrastructure.States
{
[BsonIgnoreExtraElements]
public sealed class MongoState<T, TKey> : IVersionedEntity<TKey>
public sealed class MongoState<T> : IVersionedEntity<DomainId>
{
[BsonId]
[BsonElement]
[BsonRepresentation(BsonType.String)]
public TKey DocumentId { get; set; }
public DomainId DocumentId { get; set; }
[BsonRequired]
[BsonElement]

5
backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs

@ -88,6 +88,11 @@ namespace Squidex.Infrastructure.CQRS.Events
public Task On(Envelope<IEvent> @event)
{
if (@event.Headers.Restored())
{
return Task.CompletedTask;
}
var jsonString = jsonSerializer.Serialize(@event);
var jsonBytes = Encoding.UTF8.GetBytes(jsonString);

14
backend/src/Squidex.Infrastructure/Commands/DomainObject.cs

@ -18,7 +18,7 @@ namespace Squidex.Infrastructure.Commands
{
private readonly List<Envelope<IEvent>> uncomittedEvents = new List<Envelope<IEvent>>();
private readonly SnapshotList<T> snapshots = new SnapshotList<T>();
private readonly IStore<DomainId> store;
private readonly IPersistenceFactory<T> factory;
private readonly ISemanticLog log;
private IPersistence<T>? persistence;
private bool isLoaded;
@ -45,12 +45,12 @@ namespace Squidex.Infrastructure.Commands
set => snapshots.Capacity = value;
}
protected DomainObject(IStore<DomainId> store, ISemanticLog log)
protected DomainObject(IPersistenceFactory<T> factory, ISemanticLog log)
{
Guard.NotNull(store, nameof(store));
Guard.NotNull(factory, nameof(factory));
Guard.NotNull(log, nameof(log));
this.store = store;
this.factory = factory;
this.log = log;
}
@ -68,7 +68,7 @@ namespace Squidex.Infrastructure.Commands
snapshots.Add(snapshot, snapshot.Version, false);
var allEvents = store.WithEventSourcing(GetType(), UniqueId, @event =>
var allEvents = factory.WithEventSourcing(GetType(), UniqueId, @event =>
{
var newVersion = snapshot.Version + 1;
@ -97,7 +97,7 @@ namespace Squidex.Infrastructure.Commands
{
this.uniqueId = uniqueId;
persistence = store.WithSnapshotsAndEventSourcing(GetType(), UniqueId,
persistence = factory.WithSnapshotsAndEventSourcing(GetType(), UniqueId,
new HandleSnapshot<T>(snapshot =>
{
snapshot.Version = Version + 1;
@ -175,7 +175,7 @@ namespace Squidex.Infrastructure.Commands
if (events != null)
{
var deletedId = DomainId.Combine(UniqueId, DomainId.Create("deleted"));
var deletedStream = store.WithEventSourcing(GetType(), deletedId, null);
var deletedStream = factory.WithEventSourcing(GetType(), deletedId, null);
await deletedStream.WriteEventsAsync(events);

82
backend/src/Squidex.Infrastructure/Commands/Rebuilder.cs

@ -10,10 +10,13 @@ using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Microsoft.Extensions.DependencyInjection;
using Squidex.Caching;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
#pragma warning disable RECS0108 // Warns about static fields in generic types
namespace Squidex.Infrastructure.Commands
{
public delegate Task IdSource(Func<DomainId, Task> add);
@ -21,31 +24,40 @@ namespace Squidex.Infrastructure.Commands
public class Rebuilder
{
private readonly ILocalCache localCache;
private readonly IStore<DomainId> store;
private readonly IEventStore eventStore;
private readonly IServiceProvider serviceProvider;
private static class Factory<T, TState> where T : DomainObject<TState> where TState : class, IDomainState<TState>, new()
{
private static readonly ObjectFactory ObjectFactory = ActivatorUtilities.CreateFactory(typeof(T), new[] { typeof(IPersistenceFactory<TState>) });
public static T Create(IServiceProvider serviceProvider, IPersistenceFactory<TState> persistenceFactory)
{
return (T)ObjectFactory(serviceProvider, new object[] { persistenceFactory });
}
}
public Rebuilder(
ILocalCache localCache,
IStore<DomainId> store,
IEventStore eventStore,
IServiceProvider serviceProvider)
{
Guard.NotNull(localCache, nameof(localCache));
Guard.NotNull(store, nameof(store));
Guard.NotNull(serviceProvider, nameof(serviceProvider));
Guard.NotNull(eventStore, nameof(eventStore));
this.eventStore = eventStore;
this.serviceProvider = serviceProvider;
this.localCache = localCache;
this.store = store;
}
public virtual async Task RebuildAsync<T, TState>(string filter, CancellationToken ct) where T : DomainObject<TState> where TState : class, IDomainState<TState>, new()
{
await store.GetSnapshotStore<TState>().ClearAsync();
var store = serviceProvider.GetRequiredService<IStore<TState>>();
await store.ClearSnapshotsAsync();
await InsertManyAsync<T, TState>(async target =>
await InsertManyAsync<T, TState>(store, async target =>
{
await eventStore.QueryAsync(async storedEvent =>
{
@ -60,7 +72,9 @@ namespace Squidex.Infrastructure.Commands
{
Guard.NotNull(source, nameof(source));
await InsertManyAsync<T, TState>(async target =>
var store = serviceProvider.GetRequiredService<IStore<TState>>();
await InsertManyAsync<T, TState>(store, async target =>
{
foreach (var id in source)
{
@ -69,26 +83,50 @@ namespace Squidex.Infrastructure.Commands
}, ct);
}
private async Task InsertManyAsync<T, TState>(IdSource source, CancellationToken ct = default) where T : DomainObject<TState> where TState : class, IDomainState<TState>, new()
private async Task InsertManyAsync<T, TState>(IStore<TState> store, IdSource source, CancellationToken ct = default) where T : DomainObject<TState> where TState : class, IDomainState<TState>, new()
{
var worker = new ActionBlock<DomainId>(async id =>
{
try
{
var domainObject = (T)serviceProvider.GetService(typeof(T))!;
var parallelism = Environment.ProcessorCount * 2;
domainObject.Setup(id);
const int BatchSize = 100;
await domainObject.RebuildStateAsync();
}
catch (DomainObjectNotFoundException)
var workerBlock = new ActionBlock<DomainId[]>(async ids =>
{
await using (var context = store.WithBatchContext(typeof(T)))
{
return;
await context.LoadAsync(ids);
foreach (var id in ids)
{
try
{
var domainObject = Factory<T, TState>.Create(serviceProvider, context);
domainObject.Setup(id);
await domainObject.RebuildStateAsync();
}
catch (DomainObjectNotFoundException)
{
return;
}
}
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 4
MaxDegreeOfParallelism = parallelism,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = parallelism * 2
});
var batchBlock = new BatchBlock<DomainId>(BatchSize, new GroupingDataflowBlockOptions
{
BoundedCapacity = BatchSize
});
batchBlock.LinkTo(workerBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
var handledIds = new HashSet<DomainId>();
@ -99,13 +137,13 @@ namespace Squidex.Infrastructure.Commands
{
if (handledIds.Add(id))
{
await worker.SendAsync(id, ct);
await batchBlock.SendAsync(id, ct);
}
});
worker.Complete();
batchBlock.Complete();
await worker.Completion;
await workerBlock.Completion;
}
}
}

5
backend/src/Squidex.Infrastructure/DomainId.cs

@ -15,6 +15,7 @@ namespace Squidex.Infrastructure
{
private static readonly string EmptyString = Guid.Empty.ToString();
public static readonly DomainId Empty = default;
public static readonly string IdSeparator = "--";
private readonly string? id;
@ -95,12 +96,12 @@ namespace Squidex.Infrastructure
public static DomainId Combine(NamedId<DomainId> id1, DomainId id2)
{
return new DomainId($"{id1.Id}--{id2}");
return new DomainId($"{id1.Id}{IdSeparator}{id2}");
}
public static DomainId Combine(DomainId id1, DomainId id2)
{
return new DomainId($"{id1}--{id2}");
return new DomainId($"{id1}{IdSeparator}{id2}");
}
}
}

14
backend/src/Squidex.Infrastructure/EventSourcing/CommonHeaders.cs

@ -9,16 +9,18 @@ namespace Squidex.Infrastructure.EventSourcing
{
public static class CommonHeaders
{
public static readonly string AggregateId = "AggregateId";
public static readonly string AggregateId = nameof(AggregateId);
public static readonly string CommitId = "CommitId";
public static readonly string CommitId = nameof(CommitId);
public static readonly string EventId = "EventId";
public static readonly string EventId = nameof(EventId);
public static readonly string EventNumber = "EventNumber";
public static readonly string EventNumber = nameof(EventNumber);
public static readonly string EventStreamNumber = "EventStreamNumber";
public static readonly string EventStreamNumber = nameof(EventStreamNumber);
public static readonly string Timestamp = "Timestamp";
public static readonly string Restored = nameof(Restored);
public static readonly string Timestamp = nameof(Timestamp);
}
}

36
backend/src/Squidex.Infrastructure/EventSourcing/EnvelopeExtensions.cs

@ -87,6 +87,18 @@ namespace Squidex.Infrastructure.EventSourcing
return envelope;
}
public static bool Restored(this EnvelopeHeaders headers)
{
return headers.GetBoolean(CommonHeaders.Restored);
}
public static Envelope<T> SetRestored<T>(this Envelope<T> envelope, bool value = true) where T : class, IEvent
{
envelope.Headers.Add(CommonHeaders.Restored, value);
return envelope;
}
public static long GetLong(this JsonObject obj, string key)
{
if (obj.TryGetValue(key, out var v))
@ -106,12 +118,9 @@ namespace Squidex.Infrastructure.EventSourcing
public static Guid GetGuid(this JsonObject obj, string key)
{
if (obj.TryGetValue(key, out var v))
if (obj.TryGetValue<JsonString>(key, out var v) && Guid.TryParse(v.ToString(), out var guid))
{
if (v.Type == JsonValueType.String && Guid.TryParse(v.ToString(), out var guid))
{
return guid;
}
return guid;
}
return default;
@ -119,12 +128,9 @@ namespace Squidex.Infrastructure.EventSourcing
public static Instant GetInstant(this JsonObject obj, string key)
{
if (obj.TryGetValue(key, out var v))
if (obj.TryGetValue<JsonString>(key, out var v) && InstantPattern.ExtendedIso.Parse(v.ToString()).TryGetValue(default, out var instant))
{
if (v.Type == JsonValueType.String && InstantPattern.ExtendedIso.Parse(v.ToString()).TryGetValue(default, out var instant))
{
return instant;
}
return instant;
}
return default;
@ -139,5 +145,15 @@ namespace Squidex.Infrastructure.EventSourcing
return string.Empty;
}
public static bool GetBoolean(this JsonObject obj, string key)
{
if (obj.TryGetValue<JsonBoolean>(key, out var v))
{
return v.Value;
}
return false;
}
}
}

4
backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs

@ -72,7 +72,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
{
BoundedCapacity = batchSize,
MaxDegreeOfParallelism = 1,
MaxMessagesPerTask = 1
MaxMessagesPerTask = DataflowBlockOptions.Unbounded
});
var buffer = AsyncHelper.CreateBatchBlock<Job>(batchSize, batchDelay, new GroupingDataflowBlockOptions
@ -105,7 +105,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
{
BoundedCapacity = 2,
MaxDegreeOfParallelism = 1,
MaxMessagesPerTask = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
TaskScheduler = scheduler
});

12
backend/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs

@ -35,5 +35,17 @@ namespace Squidex.Infrastructure.EventSourcing
await AppendAsync(commit.Id, commit.StreamName, commit.Offset, commit.Events);
}
}
async Task<IReadOnlyDictionary<string, IReadOnlyList<StoredEvent>>> QueryManyAsync(IEnumerable<string> streamNames)
{
var result = new Dictionary<string, IReadOnlyList<StoredEvent>>();
foreach (var streamName in streamNames)
{
result[streamName] = await QueryAsync(streamName);
}
return result;
}
}
}

4
backend/src/Squidex.Infrastructure/Json/Newtonsoft/JsonValueConverter.cs

@ -46,7 +46,7 @@ namespace Squidex.Infrastructure.Json.Newtonsoft
break;
case JsonToken.StartObject:
{
var result = JsonValue.Object();
var result = new JsonObject(1);
while (reader.Read())
{
@ -74,7 +74,7 @@ namespace Squidex.Infrastructure.Json.Newtonsoft
case JsonToken.StartArray:
{
var result = JsonValue.Array();
var result = new JsonArray(1);
while (reader.Read())
{

5
backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs

@ -56,6 +56,11 @@ namespace Squidex.Infrastructure.Json.Objects
inner = new Dictionary<string, IJsonValue>();
}
public JsonObject(int capacity)
{
inner = new Dictionary<string, IJsonValue>(capacity);
}
public JsonObject(JsonObject obj)
{
inner = new Dictionary<string, IJsonValue>(obj.inner);

14
backend/src/Squidex.Infrastructure/Orleans/GrainState.cs

@ -44,19 +44,21 @@ namespace Squidex.Infrastructure.Orleans
return Task.CompletedTask;
}
DomainId key;
if (context.GrainIdentity.PrimaryKeyString != null)
{
var store = context.ActivationServices.GetRequiredService<IStore<string>>();
persistence = store.WithSnapshots<T>(GetType(), context.GrainIdentity.PrimaryKeyString, ApplyState);
key = DomainId.Create(context.GrainIdentity.PrimaryKeyString);
}
else
{
var store = context.ActivationServices.GetRequiredService<IStore<Guid>>();
persistence = store.WithSnapshots<T>(GetType(), context.GrainIdentity.PrimaryKey, ApplyState);
key = DomainId.Create(context.GrainIdentity.PrimaryKey);
}
var factory = context.ActivationServices.GetRequiredService<IPersistenceFactory<T>>();
persistence = factory.WithSnapshots(GetType(), key, ApplyState);
return persistence.ReadAsync();
}

112
backend/src/Squidex.Infrastructure/States/BatchContext.cs

@ -0,0 +1,112 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
#pragma warning disable RECS0108 // Warns about static fields in generic types
namespace Squidex.Infrastructure.States
{
public sealed class BatchContext<T> : IBatchContext<T>
{
private static readonly List<Envelope<IEvent>> EmptyStream = new List<Envelope<IEvent>>();
private readonly Type owner;
private readonly ISnapshotStore<T> snapshotStore;
private readonly IEventStore eventStore;
private readonly IEventDataFormatter eventDataFormatter;
private readonly IStreamNameResolver streamNameResolver;
private readonly Dictionary<DomainId, (long, List<Envelope<IEvent>>)> @events = new Dictionary<DomainId, (long, List<Envelope<IEvent>>)>();
private List<(DomainId Key, T Snapshot, long Version)>? snapshots;
internal BatchContext(
Type owner,
ISnapshotStore<T> snapshotStore,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter,
IStreamNameResolver streamNameResolver)
{
this.owner = owner;
this.snapshotStore = snapshotStore;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
this.streamNameResolver = streamNameResolver;
}
internal void Add(DomainId key, T snapshot, long version)
{
snapshots ??= new List<(DomainId Key, T Snapshot, long Version)>();
snapshots.Add((key, snapshot, version));
}
public async Task LoadAsync(IEnumerable<DomainId> ids)
{
var streamNames = ids.ToDictionary(x => x, x => streamNameResolver.GetStreamName(owner, x.ToString()));
if (streamNames.Count == 0)
{
return;
}
var streams = await eventStore.QueryManyAsync(streamNames.Values);
foreach (var (id, streamName) in streamNames)
{
if (streams.TryGetValue(streamName, out var data))
{
var stream = data.Select(eventDataFormatter.ParseIfKnown).NotNull().ToList();
events[id] = (data.Count - 1, stream);
}
else
{
events[id] = (EtagVersion.Empty, EmptyStream);
}
}
}
public async ValueTask DisposeAsync()
{
await CommitAsync();
}
public Task CommitAsync()
{
var current = Interlocked.Exchange(ref snapshots, null!);
if (current == null || current.Count == 0)
{
return Task.CompletedTask;
}
return snapshotStore.WriteManyAsync(current);
}
public IPersistence<T> WithEventSourcing(Type owner, DomainId key, HandleEvent? applyEvent)
{
var (version, streamEvents) = events[key];
return new BatchPersistence<T>(key, this, version, streamEvents, applyEvent);
}
public IPersistence<T> WithSnapshotsAndEventSourcing(Type owner, DomainId key, HandleSnapshot<T>? applySnapshot, HandleEvent? applyEvent)
{
var (version, streamEvents) = events[key];
return new BatchPersistence<T>(key, this, version, streamEvents, applyEvent);
}
public IPersistence<T> WithSnapshots(Type owner, DomainId key, HandleSnapshot<T>? applySnapshot)
{
throw new NotSupportedException();
}
}
}

80
backend/src/Squidex.Infrastructure/States/BatchPersistence.cs

@ -0,0 +1,80 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.States
{
internal class BatchPersistence<T> : IPersistence<T>
{
private readonly DomainId ownerKey;
private readonly BatchContext<T> context;
private readonly IReadOnlyList<Envelope<IEvent>> events;
private readonly HandleEvent? applyEvent;
public long Version { get; }
internal BatchPersistence(DomainId ownerKey, BatchContext<T> context, long version, IReadOnlyList<Envelope<IEvent>> @events,
HandleEvent? applyEvent)
{
this.ownerKey = ownerKey;
this.context = context;
this.events = events;
this.applyEvent = applyEvent;
Version = version;
}
public Task DeleteAsync()
{
throw new NotSupportedException();
}
public Task WriteEventsAsync(IReadOnlyList<Envelope<IEvent>> events)
{
throw new NotSupportedException();
}
public Task ReadAsync(long expectedVersion = -2)
{
if (applyEvent != null)
{
foreach (var @event in events)
{
if (!applyEvent(@event))
{
break;
}
}
}
if (expectedVersion > EtagVersion.Any && expectedVersion != Version)
{
if (Version == EtagVersion.Empty)
{
throw new DomainObjectNotFoundException(ownerKey.ToString()!);
}
else
{
throw new InconsistentStateException(Version, expectedVersion);
}
}
return Task.CompletedTask;
}
public Task WriteSnapshotAsync(T state)
{
context.Add(ownerKey, state, Version);
return Task.CompletedTask;
}
}
}

16
backend/src/Squidex.Infrastructure/States/IPersistence{TState}.cs → backend/src/Squidex.Infrastructure/States/IBatchContext.cs

@ -5,22 +5,16 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.States
{
public interface IPersistence<in TState>
public interface IBatchContext<T> : IAsyncDisposable, IPersistenceFactory<T>
{
long Version { get; }
Task CommitAsync();
Task DeleteAsync();
Task WriteEventsAsync(IReadOnlyList<Envelope<IEvent>> events);
Task WriteSnapshotAsync(TState state);
Task ReadAsync(long expectedVersion = EtagVersion.Any);
Task LoadAsync(IEnumerable<DomainId> ids);
}
}
}

17
backend/src/Squidex.Infrastructure/States/IPersistence.cs

@ -1,13 +1,26 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschränkt)
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Collections.Generic;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.States
{
public interface IPersistence : IPersistence<None>
public interface IPersistence<in TState>
{
long Version { get; }
Task DeleteAsync();
Task WriteEventsAsync(IReadOnlyList<Envelope<IEvent>> events);
Task WriteSnapshotAsync(TState state);
Task ReadAsync(long expectedVersion = EtagVersion.Any);
}
}

25
backend/src/Squidex.Infrastructure/States/IPersistenceFactory.cs

@ -0,0 +1,25 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschränkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.States
{
public delegate bool HandleEvent(Envelope<IEvent> @event);
public delegate void HandleSnapshot<in T>(T state);
public interface IPersistenceFactory<T>
{
IPersistence<T> WithEventSourcing(Type owner, DomainId id, HandleEvent? applyEvent);
IPersistence<T> WithSnapshots(Type owner, DomainId id, HandleSnapshot<T>? applySnapshot);
IPersistence<T> WithSnapshotsAndEventSourcing(Type owner, DomainId id, HandleSnapshot<T>? applySnapshot, HandleEvent? applyEvent);
}
}

11
backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs

@ -6,20 +6,23 @@
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace Squidex.Infrastructure.States
{
public interface ISnapshotStore<T, in TKey>
public interface ISnapshotStore<T>
{
Task WriteAsync(TKey key, T value, long oldVersion, long newVersion);
Task WriteAsync(DomainId key, T value, long oldVersion, long newVersion);
Task<(T Value, long Version)> ReadAsync(TKey key);
Task WriteManyAsync(IEnumerable<(DomainId Key, T Value, long Version)> snapshots);
Task<(T Value, long Version)> ReadAsync(DomainId key);
Task ClearAsync();
Task RemoveAsync(TKey key);
Task RemoveAsync(DomainId key);
Task ReadAllAsync(Func<T, long, Task> callback, CancellationToken ct = default);
}

16
backend/src/Squidex.Infrastructure/States/IStore.cs

@ -6,22 +6,14 @@
// ==========================================================================
using System;
using Squidex.Infrastructure.EventSourcing;
using System.Threading.Tasks;
namespace Squidex.Infrastructure.States
{
public delegate bool HandleEvent(Envelope<IEvent> @event);
public delegate void HandleSnapshot<in T>(T state);
public interface IStore<in TKey>
public interface IStore<T> : IPersistenceFactory<T>
{
IPersistence WithEventSourcing(Type owner, TKey key, HandleEvent? applyEvent);
IPersistence<TState> WithSnapshots<TState>(Type owner, TKey key, HandleSnapshot<TState>? applySnapshot);
IPersistence<TState> WithSnapshotsAndEventSourcing<TState>(Type owner, TKey key, HandleSnapshot<TState>? applySnapshot, HandleEvent? applyEvent);
IBatchContext<T> WithBatchContext(Type owner);
ISnapshotStore<TState, TKey> GetSnapshotStore<TState>();
Task ClearSnapshotsAsync();
}
}

205
backend/src/Squidex.Infrastructure/States/Persistence.cs

@ -6,21 +6,216 @@
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
#pragma warning disable RECS0012 // 'if' statement can be re-written as 'switch' statement
namespace Squidex.Infrastructure.States
{
internal sealed class Persistence<TKey> : Persistence<None, TKey>, IPersistence where TKey : notnull
internal class Persistence<T> : IPersistence<T>
{
public Persistence(TKey ownerKey, Type ownerType,
private readonly DomainId ownerKey;
private readonly ISnapshotStore<T> snapshotStore;
private readonly IEventStore eventStore;
private readonly IEventDataFormatter eventDataFormatter;
private readonly PersistenceMode persistenceMode;
private readonly HandleSnapshot<T>? applyState;
private readonly HandleEvent? applyEvent;
private readonly Lazy<string> streamName;
private long versionSnapshot = EtagVersion.Empty;
private long versionEvents = EtagVersion.Empty;
private long version = EtagVersion.Empty;
public long Version
{
get => version;
}
private bool UseSnapshots
{
get => (persistenceMode & PersistenceMode.Snapshots) == PersistenceMode.Snapshots;
}
private bool UseEventSourcing
{
get => (persistenceMode & PersistenceMode.EventSourcing) == PersistenceMode.EventSourcing;
}
public Persistence(DomainId ownerKey, Type ownerType,
ISnapshotStore<T> snapshotStore,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter,
ISnapshotStore<None, TKey> snapshotStore,
IStreamNameResolver streamNameResolver,
PersistenceMode persistenceMode,
HandleSnapshot<T>? applyState,
HandleEvent? applyEvent)
: base(ownerKey, ownerType, eventStore, eventDataFormatter, snapshotStore, streamNameResolver,
PersistenceMode.EventSourcing, null, applyEvent)
{
this.ownerKey = ownerKey;
this.applyState = applyState;
this.applyEvent = applyEvent;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
this.persistenceMode = persistenceMode;
this.snapshotStore = snapshotStore;
streamName = new Lazy<string>(() => streamNameResolver.GetStreamName(ownerType, ownerKey.ToString()!));
}
public async Task DeleteAsync()
{
if (UseSnapshots)
{
await snapshotStore.RemoveAsync(ownerKey);
}
if (UseEventSourcing)
{
await eventStore.DeleteStreamAsync(streamName.Value);
}
}
public async Task ReadAsync(long expectedVersion = EtagVersion.Any)
{
versionSnapshot = EtagVersion.Empty;
versionEvents = EtagVersion.Empty;
if (UseSnapshots)
{
await ReadSnapshotAsync();
}
if (UseEventSourcing)
{
await ReadEventsAsync();
}
UpdateVersion();
if (expectedVersion > EtagVersion.Any && expectedVersion != version)
{
if (version == EtagVersion.Empty)
{
throw new DomainObjectNotFoundException(ownerKey.ToString()!);
}
else
{
throw new InconsistentStateException(version, expectedVersion);
}
}
}
private async Task ReadSnapshotAsync()
{
var (state, position) = await snapshotStore.ReadAsync(ownerKey);
// Treat all negative values as not-found (empty).
position = Math.Max(position, EtagVersion.Empty);
versionSnapshot = position;
versionEvents = position;
if (applyState != null && position >= 0)
{
applyState(state);
}
}
private async Task ReadEventsAsync()
{
var events = await eventStore.QueryAsync(streamName.Value, versionEvents + 1);
var isStopped = false;
foreach (var @event in events)
{
var newVersion = versionEvents + 1;
if (@event.EventStreamNumber != newVersion)
{
throw new InvalidOperationException("Events must follow the snapshot version in consecutive order with no gaps.");
}
// Skip the parsing for performance reasons if we are not interested, but continue reading to get the version
if (!isStopped)
{
var parsedEvent = eventDataFormatter.ParseIfKnown(@event);
if (applyEvent != null && parsedEvent != null)
{
isStopped = !applyEvent(parsedEvent);
}
}
versionEvents++;
}
}
public async Task WriteSnapshotAsync(T state)
{
var oldVersion = versionSnapshot;
if (oldVersion == EtagVersion.Empty && UseEventSourcing)
{
oldVersion = (versionEvents - 1);
}
var newVersion = UseEventSourcing ? versionEvents : oldVersion + 1;
if (newVersion == versionSnapshot)
{
return;
}
await snapshotStore.WriteAsync(ownerKey, state, oldVersion, newVersion);
versionSnapshot = newVersion;
UpdateVersion();
}
public async Task WriteEventsAsync(IReadOnlyList<Envelope<IEvent>> events)
{
Guard.NotEmpty(events, nameof(events));
var oldVersion = EtagVersion.Any;
if (UseEventSourcing)
{
oldVersion = versionEvents;
}
var eventCommitId = Guid.NewGuid();
var eventData = events.Select(x => eventDataFormatter.ToEventData(x, eventCommitId, true)).ToArray();
try
{
await eventStore.AppendAsync(eventCommitId, streamName.Value, oldVersion, eventData);
}
catch (WrongEventVersionException ex)
{
throw new InconsistentStateException(ex.CurrentVersion, ex.ExpectedVersion, ex);
}
versionEvents += eventData.Length;
}
private void UpdateVersion()
{
if (persistenceMode == PersistenceMode.Snapshots)
{
version = versionSnapshot;
}
else if (persistenceMode == PersistenceMode.EventSourcing)
{
version = versionEvents;
}
else if (persistenceMode == PersistenceMode.SnapshotsAndEventSourcing)
{
version = Math.Max(versionEvents, versionSnapshot);
}
}
}
}

221
backend/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs

@ -1,221 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschränkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
#pragma warning disable RECS0012 // 'if' statement can be re-written as 'switch' statement
namespace Squidex.Infrastructure.States
{
internal class Persistence<TSnapshot, TKey> : IPersistence<TSnapshot> where TKey : notnull
{
private readonly TKey ownerKey;
private readonly ISnapshotStore<TSnapshot, TKey> snapshotStore;
private readonly IEventStore eventStore;
private readonly IEventDataFormatter eventDataFormatter;
private readonly PersistenceMode persistenceMode;
private readonly HandleSnapshot<TSnapshot>? applyState;
private readonly HandleEvent? applyEvent;
private readonly Lazy<string> streamName;
private long versionSnapshot = EtagVersion.Empty;
private long versionEvents = EtagVersion.Empty;
private long version = EtagVersion.Empty;
public long Version
{
get => version;
}
private bool UseSnapshots
{
get => (persistenceMode & PersistenceMode.Snapshots) == PersistenceMode.Snapshots;
}
private bool UseEventSourcing
{
get => (persistenceMode & PersistenceMode.EventSourcing) == PersistenceMode.EventSourcing;
}
public Persistence(TKey ownerKey, Type ownerType,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter,
ISnapshotStore<TSnapshot, TKey> snapshotStore,
IStreamNameResolver streamNameResolver,
PersistenceMode persistenceMode,
HandleSnapshot<TSnapshot>? applyState,
HandleEvent? applyEvent)
{
this.ownerKey = ownerKey;
this.applyState = applyState;
this.applyEvent = applyEvent;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
this.persistenceMode = persistenceMode;
this.snapshotStore = snapshotStore;
streamName = new Lazy<string>(() => streamNameResolver.GetStreamName(ownerType, ownerKey.ToString()!));
}
public async Task DeleteAsync()
{
if (UseSnapshots)
{
await snapshotStore.RemoveAsync(ownerKey);
}
if (UseEventSourcing)
{
await eventStore.DeleteStreamAsync(streamName.Value);
}
}
public async Task ReadAsync(long expectedVersion = EtagVersion.Any)
{
versionSnapshot = EtagVersion.Empty;
versionEvents = EtagVersion.Empty;
if (UseSnapshots)
{
await ReadSnapshotAsync();
}
if (UseEventSourcing)
{
await ReadEventsAsync();
}
UpdateVersion();
if (expectedVersion > EtagVersion.Any && expectedVersion != version)
{
if (version == EtagVersion.Empty)
{
throw new DomainObjectNotFoundException(ownerKey.ToString()!);
}
else
{
throw new InconsistentStateException(version, expectedVersion);
}
}
}
private async Task ReadSnapshotAsync()
{
var (state, position) = await snapshotStore.ReadAsync(ownerKey);
// Treat all negative values as not-found (empty).
position = Math.Max(position, EtagVersion.Empty);
versionSnapshot = position;
versionEvents = position;
if (applyState != null && position >= 0)
{
applyState(state);
}
}
private async Task ReadEventsAsync()
{
var events = await eventStore.QueryAsync(streamName.Value, versionEvents + 1);
var isStopped = false;
foreach (var @event in events)
{
var newVersion = versionEvents + 1;
if (@event.EventStreamNumber != newVersion)
{
throw new InvalidOperationException("Events must follow the snapshot version in consecutive order with no gaps.");
}
// Skip the parsing for performance reasons if we are not interested, but continue reading to get the version.
if (!isStopped)
{
var parsedEvent = eventDataFormatter.ParseIfKnown(@event);
if (applyEvent != null && parsedEvent != null)
{
isStopped = !applyEvent(parsedEvent);
}
}
versionEvents++;
}
}
public async Task WriteSnapshotAsync(TSnapshot state)
{
var oldVersion = versionSnapshot;
if (oldVersion == EtagVersion.Empty && UseEventSourcing)
{
oldVersion = (versionEvents - 1);
}
var newVersion = UseEventSourcing ? versionEvents : oldVersion + 1;
if (newVersion == versionSnapshot)
{
return;
}
await snapshotStore.WriteAsync(ownerKey, state, oldVersion, newVersion);
versionSnapshot = newVersion;
UpdateVersion();
}
public async Task WriteEventsAsync(IReadOnlyList<Envelope<IEvent>> events)
{
Guard.NotEmpty(events, nameof(events));
var oldVersion = EtagVersion.Any;
if (UseEventSourcing)
{
oldVersion = versionEvents;
}
var eventCommitId = Guid.NewGuid();
var eventData = events.Select(x => eventDataFormatter.ToEventData(x, eventCommitId, true)).ToArray();
try
{
await eventStore.AppendAsync(eventCommitId, streamName.Value, oldVersion, eventData);
}
catch (WrongEventVersionException ex)
{
throw new InconsistentStateException(ex.CurrentVersion, ex.ExpectedVersion, ex);
}
versionEvents += eventData.Length;
}
private void UpdateVersion()
{
if (persistenceMode == PersistenceMode.Snapshots)
{
version = versionSnapshot;
}
else if (persistenceMode == PersistenceMode.EventSourcing)
{
version = versionEvents;
}
else if (persistenceMode == PersistenceMode.SnapshotsAndEventSourcing)
{
version = Math.Max(versionEvents, versionSnapshot);
}
}
}
}

59
backend/src/Squidex.Infrastructure/States/Store.cs

@ -6,73 +6,76 @@
// ==========================================================================
using System;
using System.Threading.Tasks;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.States
{
public sealed class Store<TKey> : IStore<TKey> where TKey : notnull
public sealed class Store<T> : IStore<T>
{
private readonly IServiceProvider services;
private readonly IStreamNameResolver streamNameResolver;
private readonly ISnapshotStore<T> snapshotStore;
private readonly IEventStore eventStore;
private readonly IEventDataFormatter eventDataFormatter;
public Store(
ISnapshotStore<T> snapshotStore,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter,
IServiceProvider services,
IStreamNameResolver streamNameResolver)
{
Guard.NotNull(snapshotStore, nameof(snapshotStore));
Guard.NotNull(eventStore, nameof(eventStore));
Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter));
Guard.NotNull(services, nameof(services));
Guard.NotNull(streamNameResolver, nameof(streamNameResolver));
this.snapshotStore = snapshotStore;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
this.services = services;
this.streamNameResolver = streamNameResolver;
}
public IPersistence WithEventSourcing(Type owner, TKey key, HandleEvent? applyEvent)
public Task ClearSnapshotsAsync()
{
return CreatePersistence(owner, key, applyEvent);
return snapshotStore.ClearAsync();
}
public IPersistence<TState> WithSnapshots<TState>(Type owner, TKey key, HandleSnapshot<TState>? applySnapshot)
public IBatchContext<T> WithBatchContext(Type owner)
{
return CreatePersistence(owner, key, PersistenceMode.Snapshots, applySnapshot, null);
return new BatchContext<T>(owner,
snapshotStore,
eventStore,
eventDataFormatter,
streamNameResolver);
}
public IPersistence<TState> WithSnapshotsAndEventSourcing<TState>(Type owner, TKey key, HandleSnapshot<TState>? applySnapshot, HandleEvent? applyEvent)
public IPersistence<T> WithEventSourcing(Type owner, DomainId key, HandleEvent? applyEvent)
{
return CreatePersistence(owner, key, PersistenceMode.SnapshotsAndEventSourcing,
applySnapshot, applyEvent);
return CreatePersistence(owner, key, PersistenceMode.EventSourcing, null, applyEvent);
}
private IPersistence CreatePersistence(Type owner, TKey key, HandleEvent? applyEvent)
public IPersistence<T> WithSnapshots(Type owner, DomainId key, HandleSnapshot<T>? applySnapshot)
{
Guard.NotNull(key, nameof(key));
var snapshotStore = GetSnapshotStore<None>();
return new Persistence<TKey>(key, owner, eventStore, eventDataFormatter,
snapshotStore, streamNameResolver, applyEvent);
return CreatePersistence(owner, key, PersistenceMode.Snapshots, applySnapshot, null);
}
private IPersistence<TState> CreatePersistence<TState>(Type owner, TKey key, PersistenceMode mode, HandleSnapshot<TState>? applySnapshot, HandleEvent? applyEvent)
public IPersistence<T> WithSnapshotsAndEventSourcing(Type owner, DomainId key, HandleSnapshot<T>? applySnapshot, HandleEvent? applyEvent)
{
Guard.NotNull(key, nameof(key));
var snapshotStore = GetSnapshotStore<TState>();
return new Persistence<TState, TKey>(key, owner, eventStore, eventDataFormatter,
snapshotStore, streamNameResolver, mode, applySnapshot, applyEvent);
return CreatePersistence(owner, key, PersistenceMode.SnapshotsAndEventSourcing, applySnapshot, applyEvent);
}
public ISnapshotStore<TState, TKey> GetSnapshotStore<TState>()
private IPersistence<T> CreatePersistence(Type owner, DomainId key, PersistenceMode mode, HandleSnapshot<T>? applySnapshot, HandleEvent? applyEvent)
{
return (ISnapshotStore<TState, TKey>)services.GetService(typeof(ISnapshotStore<TState, TKey>))!;
Guard.NotNull(key, nameof(key));
return new Persistence<T>(key, owner,
snapshotStore,
eventStore,
eventDataFormatter,
streamNameResolver,
mode,
applySnapshot,
applyEvent);
}
}
}

17
backend/src/Squidex.Infrastructure/States/StoreExtensions.cs

@ -16,22 +16,5 @@ namespace Squidex.Infrastructure.States
{
return persistence.WriteEventsAsync(new[] { @event });
}
public static Task ClearSnapshotsAsync<TKey, TSnapshot>(this IStore<TKey> store)
{
return store.GetSnapshotStore<TSnapshot>().ClearAsync();
}
public static Task RemoveSnapshotAsync<TKey, TSnapshot>(this IStore<TKey> store, TKey key)
{
return store.GetSnapshotStore<TSnapshot>().RemoveAsync(key);
}
public static async Task<TSnapshot> GetSnapshotAsync<TKey, TSnapshot>(this IStore<TKey> store, TKey key)
{
var result = await store.GetSnapshotStore<TSnapshot>().ReadAsync(key);
return result.Value;
}
}
}

4
backend/src/Squidex.Infrastructure/Tasks/PartitionedActionBlock.cs

@ -42,7 +42,7 @@ namespace Squidex.Infrastructure.Tasks
var workerOption = SimpleMapper.Map(dataflowBlockOptions, new ExecutionDataflowBlockOptions());
workerOption.MaxDegreeOfParallelism = 1;
workerOption.MaxMessagesPerTask = 1;
workerOption.MaxMessagesPerTask = DataflowBlockOptions.Unbounded;
workers[i] = new ActionBlock<TInput>(action, workerOption);
}
@ -50,7 +50,7 @@ namespace Squidex.Infrastructure.Tasks
var distributorOption = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
MaxMessagesPerTask = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = 1
};

10
backend/src/Squidex/Config/Domain/StoreServices.cs

@ -53,7 +53,7 @@ namespace Squidex.Config.Domain
var mongoDatabaseName = config.GetRequiredValue("store:mongoDb:database");
var mongoContentDatabaseName = config.GetOptionalValue("store:mongoDb:contentDatabase", mongoDatabaseName);
services.AddSingleton(typeof(ISnapshotStore<,>), typeof(MongoSnapshotStore<,>));
services.AddSingleton(typeof(ISnapshotStore<>), typeof(MongoSnapshotStore<>));
services.AddSingletonAs(c => GetClient(mongoConfiguration))
.As<IMongoClient>();
@ -110,13 +110,13 @@ namespace Squidex.Config.Domain
.As<IUserStore<IdentityUser>>().As<IUserFactory>();
services.AddSingletonAs<MongoAssetRepository>()
.As<IAssetRepository>().As<ISnapshotStore<AssetDomainObject.State, DomainId>>();
.As<IAssetRepository>().As<ISnapshotStore<AssetDomainObject.State>>();
services.AddSingletonAs<MongoAssetFolderRepository>()
.As<IAssetFolderRepository>().As<ISnapshotStore<AssetFolderDomainObject.State, DomainId>>();
.As<IAssetFolderRepository>().As<ISnapshotStore<AssetFolderDomainObject.State>>();
services.AddSingletonAs(c => ActivatorUtilities.CreateInstance<MongoContentRepository>(c, GetDatabase(c, mongoContentDatabaseName), false))
.As<IContentRepository>().As<ISnapshotStore<ContentDomainObject.State, DomainId>>();
.As<IContentRepository>().As<ISnapshotStore<ContentDomainObject.State>>();
services.AddSingletonAs<MongoSchemasHash>()
.AsOptional<ISchemasHash>().As<IEventConsumer>();
@ -138,6 +138,8 @@ namespace Squidex.Config.Domain
});
services.AddSingleton(typeof(IStore<>), typeof(Store<>));
services.AddSingleton(typeof(IPersistenceFactory<>), typeof(Store<>));
}
private static IMongoClient GetClient(string configuration)

2
backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs

@ -69,7 +69,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
{ patternId2, new AppPattern("Numbers", "[0-9]*") }
};
sut = new AppDomainObject(Store, A.Dummy<ISemanticLog>(), initialPatterns, appPlansProvider, appPlansBillingManager, userResolver);
sut = new AppDomainObject(PersistenceFactory, A.Dummy<ISemanticLog>(), initialPatterns, appPlansProvider, appPlansBillingManager, userResolver);
sut.Setup(Id);
}

11
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetPermanentDeleterTests.cs

@ -54,6 +54,17 @@ namespace Squidex.Domain.Apps.Entities.Assets
Assert.Equal(nameof(AssetPermanentDeleter), consumer.Name);
}
[Fact]
public async Task Should_not_delete_assets_when_event_restored()
{
var @event = new AssetDeleted { AppId = appId, AssetId = DomainId.NewGuid() };
await sut.On(Envelope.Create(@event).SetRestored());
A.CallTo(() => assetFiletore.DeleteAsync(appId.Id, @event.AssetId, A<long>._))
.MustNotHaveHappened();
}
[Fact]
public async Task Should_delete_assets_for_all_versions()
{

2
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs

@ -46,7 +46,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
A.CallTo(() => tagService.NormalizeTagsAsync(AppId, TagGroups.Assets, A<HashSet<string>>._, A<HashSet<string>>._))
.ReturnsLazily(x => Task.FromResult(x.GetArgument<HashSet<string>>(2)?.ToDictionary(x => x) ?? new Dictionary<string, string>()));
sut = new AssetDomainObject(Store, A.Dummy<ISemanticLog>(), tagService, assetQuery, contentRepository);
sut = new AssetDomainObject(PersistenceFactory, A.Dummy<ISemanticLog>(), tagService, assetQuery, contentRepository);
sut.Setup(Id);
}

2
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetFolderDomainObjectTests.cs

@ -34,7 +34,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
A.CallTo(() => assetQuery.FindAssetFolderAsync(AppId, parentId))
.Returns(new List<IAssetFolderEntity> { A.Fake<IAssetFolderEntity>() });
sut = new AssetFolderDomainObject(Store, A.Dummy<ISemanticLog>(), assetQuery);
sut = new AssetFolderDomainObject(PersistenceFactory, A.Dummy<ISemanticLog>(), assetQuery);
sut.Setup(Id);
}

11
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs

@ -61,6 +61,17 @@ namespace Squidex.Domain.Apps.Entities.Assets
Assert.Equal(nameof(RecursiveDeleter), consumer.Name);
}
[Fact]
public async Task Should_Not_invoke_delete_commands_when_event_restored()
{
var @event = new AssetFolderDeleted { AppId = appId, AssetFolderId = DomainId.NewGuid() };
await sut.On(Envelope.Create(@event).SetRestored());
A.CallTo(() => commandBus.PublishAsync(A<ICommand>._))
.MustNotHaveHappened();
}
[Fact]
public async Task Should_invoke_delete_commands_for_all_subfolders()
{

4
backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupCompatibilityTests.cs

@ -49,14 +49,14 @@ namespace Squidex.Domain.Apps.Entities.Backup
}
[Fact]
public async Task Should_throw_exception_if_backup_has_no_version()
public async Task Should_not_throw_exception_if_backup_has_no_version()
{
var reader = A.Fake<IBackupReader>();
A.CallTo(() => reader.ReadJsonAsync<CompatibilityExtensions.FileVersion>(A<string>._))
.Throws(new FileNotFoundException());
await Assert.ThrowsAsync<BackupRestoreException>(() => reader.CheckCompatibilityAsync());
await reader.CheckCompatibilityAsync();
}
}
}

70
backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/StreamMapperTests.cs

@ -0,0 +1,70 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using FakeItEasy;
using Squidex.Infrastructure;
using Xunit;
namespace Squidex.Domain.Apps.Entities.Backup
{
public class StreamMapperTests
{
private readonly DomainId appIdOld = DomainId.NewGuid();
private readonly DomainId appId = DomainId.NewGuid();
private readonly StreamMapper sut;
public StreamMapperTests()
{
sut = new StreamMapper(new RestoreContext(appId,
A.Fake<IUserMapping>(),
A.Fake<IBackupReader>(),
appIdOld));
}
[Fact]
public void Should_map_old_app_id()
{
var result = sut.Map($"app-{appIdOld}");
Assert.Equal(($"app-{appId}", appId), result);
}
[Fact]
public void Should_map_old_app_broken_id()
{
var result = sut.Map($"app-{appIdOld}--{appIdOld}");
Assert.Equal(($"app-{appId}", appId), result);
}
[Fact]
public void Should_map_non_app_id()
{
var result = sut.Map($"content-{appIdOld}--123");
Assert.Equal(($"content-{appId}--123", DomainId.Create($"{appId}--123")), result);
}
[Fact]
public void Should_map_non_app_id_with_double_slash()
{
var result = sut.Map($"content-{appIdOld}--other--id");
Assert.Equal(($"content-{appId}--other--id", DomainId.Create($"{appId}--other--id")), result);
}
[Fact]
public void Should_map_non_combined_id()
{
var id = DomainId.NewGuid();
var result = sut.Map($"content-{id}");
Assert.Equal(($"content-{appId}--{id}", DomainId.Create($"{appId}--{id}")), result);
}
}
}

3
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/DomainObject/ContentDomainObjectTests.cs

@ -5,7 +5,6 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Linq;
using System.Threading.Tasks;
using FakeItEasy;
using Microsoft.Extensions.DependencyInjection;
@ -119,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.DomainObject
.AddSingleton<IValidatorsFactory>(new DefaultValidatorsFactory())
.BuildServiceProvider();
sut = new ContentDomainObject(Store, log, serviceProvider);
sut = new ContentDomainObject(PersistenceFactory, log, serviceProvider);
sut.Setup(Id);
}

2
backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/DomainObject/RuleDomainObjectTests.cs

@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Rules.DomainObject
public RuleDomainObjectTests()
{
sut = new RuleDomainObject(Store, A.Dummy<ISemanticLog>(), appProvider, ruleEnqueuer);
sut = new RuleDomainObject(PersistenceFactory, A.Dummy<ISemanticLog>(), appProvider, ruleEnqueuer);
sut.Setup(Id);
}

34
backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs

@ -97,11 +97,36 @@ namespace Squidex.Domain.Apps.Entities.Rules
{
var @event = Envelope.Create<IEvent>(new ContentCreated { AppId = appId });
var rule1 = CreateRule();
var rule2 = CreateRule();
var job1 = new RuleJob { Created = now };
SetupRules(@event, job1);
await sut.On(@event);
A.CallTo(() => ruleEventRepository.EnqueueAsync(job1, (Exception?)null))
.MustHaveHappened();
}
[Fact]
public async Task Should_not_eqneue_when_event_restored()
{
var @event = Envelope.Create<IEvent>(new ContentCreated { AppId = appId });
var job1 = new RuleJob { Created = now };
SetupRules(@event, job1);
await sut.On(@event.SetRestored(true));
A.CallTo(() => ruleEventRepository.EnqueueAsync(A<RuleJob>._, A<Exception?>._))
.MustNotHaveHappened();
}
private void SetupRules(Envelope<IEvent> @event, RuleJob job1)
{
var rule1 = CreateRule();
var rule2 = CreateRule();
A.CallTo(() => appProvider.GetRulesAsync(appId.Id))
.Returns(new List<IRuleEntity> { rule1, rule2 });
@ -110,11 +135,6 @@ namespace Squidex.Domain.Apps.Entities.Rules
A.CallTo(() => ruleService.CreateJobsAsync(rule2.RuleDef, rule2.Id, @event, true))
.Returns(new List<(RuleJob, Exception?)>());
await sut.On(@event);
A.CallTo(() => ruleEventRepository.EnqueueAsync(job1, (Exception?)null))
.MustHaveHappened();
}
private static RuleEntity CreateRule()

2
backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/DomainObject/SchemaDomainObjectTests.cs

@ -37,7 +37,7 @@ namespace Squidex.Domain.Apps.Entities.Schemas.DomainObject
public SchemaDomainObjectTests()
{
sut = new SchemaDomainObject(Store, A.Dummy<ISemanticLog>());
sut = new SchemaDomainObject(PersistenceFactory, A.Dummy<ISemanticLog>());
sut.Setup(Id);
}

21
backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs

@ -23,9 +23,8 @@ namespace Squidex.Domain.Apps.Entities.TestHelpers
{
public abstract class HandlerTestBase<TState>
{
private readonly IStore<DomainId> store = A.Fake<IStore<DomainId>>();
private readonly IPersistence<TState> persistenceWithState = A.Fake<IPersistence<TState>>();
private readonly IPersistence persistence = A.Fake<IPersistence>();
private readonly IPersistenceFactory<TState> persistenceFactory = A.Fake<IStore<TState>>();
private readonly IPersistence<TState> persistence = A.Fake<IPersistence<TState>>();
protected RefToken Actor { get; } = RefToken.User("me");
@ -53,30 +52,24 @@ namespace Squidex.Domain.Apps.Entities.TestHelpers
protected abstract DomainId Id { get; }
public IStore<DomainId> Store
public IPersistenceFactory<TState> PersistenceFactory
{
get => store;
get => persistenceFactory;
}
public IEnumerable<Envelope<IEvent>> LastEvents { get; private set; } = Enumerable.Empty<Envelope<IEvent>>();
protected HandlerTestBase()
{
A.CallTo(() => store.WithSnapshotsAndEventSourcing(A<Type>._, Id, A<HandleSnapshot<TState>>._, A<HandleEvent>._))
.Returns(persistenceWithState);
A.CallTo(() => store.WithEventSourcing(A<Type>._, Id, A<HandleEvent>._))
A.CallTo(() => persistenceFactory.WithSnapshotsAndEventSourcing(A<Type>._, Id, A<HandleSnapshot<TState>>._, A<HandleEvent>._))
.Returns(persistence);
A.CallTo(() => persistenceWithState.WriteEventsAsync(A<IReadOnlyList<Envelope<IEvent>>>._))
.Invokes((IReadOnlyList<Envelope<IEvent>> events) => LastEvents = events);
A.CallTo(() => persistenceFactory.WithEventSourcing(A<Type>._, Id, A<HandleEvent>._))
.Returns(persistence);
A.CallTo(() => persistence.WriteEventsAsync(A<IReadOnlyList<Envelope<IEvent>>>._))
.Invokes((IReadOnlyList<Envelope<IEvent>> events) => LastEvents = events);
A.CallTo(() => persistenceWithState.DeleteAsync())
.Invokes(() => LastEvents = Enumerable.Empty<Envelope<IEvent>>());
A.CallTo(() => persistence.DeleteAsync())
.Invokes(() => LastEvents = Enumerable.Empty<Envelope<IEvent>>());
}

16
backend/tests/Squidex.Domain.Users.Tests/DefaultKeyStoreTests.cs

@ -5,9 +5,9 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Threading.Tasks;
using FakeItEasy;
using Squidex.Infrastructure;
using Squidex.Infrastructure.States;
using Xunit;
@ -15,7 +15,7 @@ namespace Squidex.Domain.Users
{
public class DefaultKeyStoreTests
{
private readonly ISnapshotStore<DefaultKeyStore.State, Guid> store = A.Fake<ISnapshotStore<DefaultKeyStore.State, Guid>>();
private readonly ISnapshotStore<DefaultKeyStore.State> store = A.Fake<ISnapshotStore<DefaultKeyStore.State>>();
private readonly DefaultKeyStore sut;
public DefaultKeyStoreTests()
@ -26,7 +26,7 @@ namespace Squidex.Domain.Users
[Fact]
public async Task Should_generate_signing_credentials_once()
{
A.CallTo(() => store.ReadAsync(A<Guid>._))
A.CallTo(() => store.ReadAsync(A<DomainId>._))
.Returns((null!, 0));
var credentials1 = await sut.GetSigningCredentialsAsync();
@ -34,17 +34,17 @@ namespace Squidex.Domain.Users
Assert.Same(credentials1, credentials2);
A.CallTo(() => store.ReadAsync(A<Guid>._))
A.CallTo(() => store.ReadAsync(A<DomainId>._))
.MustHaveHappenedOnceExactly();
A.CallTo(() => store.WriteAsync(A<Guid>._, A<DefaultKeyStore.State>._, 0, 0))
A.CallTo(() => store.WriteAsync(A<DomainId>._, A<DefaultKeyStore.State>._, 0, 0))
.MustHaveHappenedOnceExactly();
}
[Fact]
public async Task Should_generate_validation_keys_once()
{
A.CallTo(() => store.ReadAsync(A<Guid>._))
A.CallTo(() => store.ReadAsync(A<DomainId>._))
.Returns((null!, 0));
var credentials1 = await sut.GetValidationKeysAsync();
@ -52,10 +52,10 @@ namespace Squidex.Domain.Users
Assert.Same(credentials1, credentials2);
A.CallTo(() => store.ReadAsync(A<Guid>._))
A.CallTo(() => store.ReadAsync(A<DomainId>._))
.MustHaveHappenedOnceExactly();
A.CallTo(() => store.WriteAsync(A<Guid>._, A<DefaultKeyStore.State>._, 0, 0))
A.CallTo(() => store.WriteAsync(A<DomainId>._, A<DefaultKeyStore.State>._, 0, 0))
.MustHaveHappenedOnceExactly();
}
}

5
backend/tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs

@ -10,6 +10,7 @@ using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using FakeItEasy;
using Squidex.Infrastructure;
using Squidex.Infrastructure.States;
using Xunit;
@ -17,7 +18,7 @@ namespace Squidex.Domain.Users
{
public sealed class DefaultXmlRepositoryTests
{
private readonly ISnapshotStore<DefaultXmlRepository.State, string> store = A.Fake<ISnapshotStore<DefaultXmlRepository.State, string>>();
private readonly ISnapshotStore<DefaultXmlRepository.State> store = A.Fake<ISnapshotStore<DefaultXmlRepository.State>>();
private readonly DefaultXmlRepository sut;
public DefaultXmlRepositoryTests()
@ -54,7 +55,7 @@ namespace Squidex.Domain.Users
sut.StoreElement(xml, "name");
A.CallTo(() => store.WriteAsync("name", A<DefaultXmlRepository.State>._, A<long>._, 0))
A.CallTo(() => store.WriteAsync(DomainId.Create("name"), A<DefaultXmlRepository.State>._, A<long>._, 0))
.MustHaveHappened();
}
}

20
backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs

@ -18,14 +18,14 @@ namespace Squidex.Infrastructure.Commands
{
public class DomainObjectTests
{
private readonly IStore<DomainId> store = A.Fake<IStore<DomainId>>();
private readonly IPersistenceFactory<MyDomainState> persistenceFactory = A.Fake<IPersistenceFactory<MyDomainState>>();
private readonly IPersistence<MyDomainState> persistence = A.Fake<IPersistence<MyDomainState>>();
private readonly DomainId id = DomainId.NewGuid();
private readonly MyDomainObject sut;
public DomainObjectTests()
{
sut = new MyDomainObject(store);
sut = new MyDomainObject(persistenceFactory);
}
[Fact]
@ -338,9 +338,9 @@ namespace Squidex.Infrastructure.Commands
SetupCreated(4);
SetupDeleted();
var deleteStream = A.Fake<IPersistence>();
var deleteStream = A.Fake<IPersistence<MyDomainState>>();
A.CallTo(() => store.WithEventSourcing(typeof(MyDomainObject), DomainId.Combine(id, DomainId.Create("deleted")), null))
A.CallTo(() => persistenceFactory.WithEventSourcing(typeof(MyDomainObject), DomainId.Combine(id, DomainId.Create("deleted")), null))
.Returns(deleteStream);
await sut.ExecuteAsync(new DeletePermanent());
@ -376,7 +376,7 @@ namespace Squidex.Infrastructure.Commands
AssertSnapshot(version_0, 3, 0);
AssertSnapshot(version_1, 4, 1);
A.CallTo(() => store.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>._))
A.CallTo(() => persistenceFactory.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>._))
.MustNotHaveHappened();
}
@ -400,7 +400,7 @@ namespace Squidex.Infrastructure.Commands
AssertSnapshot(version_0, 3, 0);
AssertSnapshot(version_1, 4, 1);
A.CallTo(() => store.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>._))
A.CallTo(() => persistenceFactory.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>._))
.MustHaveHappened();
}
@ -428,7 +428,7 @@ namespace Squidex.Infrastructure.Commands
handleEvent(Envelope.Create(new ValueChanged { Value = value }));
});
A.CallTo(() => store.WithSnapshotsAndEventSourcing(typeof(MyDomainObject), id, A<HandleSnapshot<MyDomainState>>._, A<HandleEvent>._))
A.CallTo(() => persistenceFactory.WithSnapshotsAndEventSourcing(typeof(MyDomainObject), id, A<HandleSnapshot<MyDomainState>>._, A<HandleEvent>._))
.Invokes(args =>
{
handleEvent = args.GetArgument<HandleEvent>(3)!;
@ -450,9 +450,9 @@ namespace Squidex.Infrastructure.Commands
A.CallTo(() => persistence.WriteEventsAsync(A<IReadOnlyList<Envelope<IEvent>>>._))
.Invokes(c => @events.AddRange(c.GetArgument<IReadOnlyList<Envelope<IEvent>>>(0)!));
var eventsPersistence = A.Fake<IPersistence>();
var eventsPersistence = A.Fake<IPersistence<MyDomainState>>();
A.CallTo(() => store.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>._))
A.CallTo(() => persistenceFactory.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>._))
.Invokes(args =>
{
handleEvent = args.GetArgument<HandleEvent>(2)!;
@ -471,7 +471,7 @@ namespace Squidex.Infrastructure.Commands
private void SetupEmpty()
{
A.CallTo(() => store.WithSnapshotsAndEventSourcing(typeof(MyDomainObject), id, A<HandleSnapshot<MyDomainState>>._, A<HandleEvent>._))
A.CallTo(() => persistenceFactory.WithSnapshotsAndEventSourcing(typeof(MyDomainObject), id, A<HandleSnapshot<MyDomainState>>._, A<HandleEvent>._))
.Returns(persistence);
A.CallTo(() => persistence.Version)

88
backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs

@ -62,8 +62,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await Assert.ThrowsAsync<WrongEventVersionException>(() => Sut.AppendAsync(Guid.NewGuid(), streamName, 0, events));
@ -76,8 +76,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await Sut.AppendAsync(Guid.NewGuid(), streamName, events);
@ -92,8 +92,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await Sut.AppendAsync(Guid.NewGuid(), streamName, events);
@ -112,14 +112,14 @@ namespace Squidex.Infrastructure.EventSourcing
}
[Fact]
public async Task Should_append_event_unsafe()
public async Task Should_append_events_unsafe()
{
var streamName = $"test-{Guid.NewGuid()}";
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await Sut.AppendUnsafeAsync(new List<EventCommit>
@ -147,8 +147,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
var readEvents = await QueryWithSubscriptionAsync(streamName, async () =>
@ -172,8 +172,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events1 = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await QueryWithSubscriptionAsync(streamName, async () =>
@ -183,8 +183,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events2 = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
var readEventsFromPosition = await QueryWithSubscriptionAsync(streamName, async () =>
@ -219,8 +219,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await Sut.AppendAsync(Guid.NewGuid(), streamName, events);
@ -239,6 +239,45 @@ namespace Squidex.Infrastructure.EventSourcing
ShouldBeEquivalentTo(readEvents2, expected);
}
[Fact]
public async Task Should_read_multiple_streams()
{
var streamName1 = $"test-{Guid.NewGuid()}";
var streamName2 = $"test-{Guid.NewGuid()}";
var events1 = new[]
{
CreateEventData(1),
CreateEventData(2)
};
var events2 = new[]
{
CreateEventData(3),
CreateEventData(4)
};
await Sut.AppendAsync(Guid.NewGuid(), streamName1, events1);
await Sut.AppendAsync(Guid.NewGuid(), streamName2, events2);
var readEvents = await Sut.QueryManyAsync(new[] { streamName1, streamName2 });
var expected1 = new[]
{
new StoredEvent(streamName1, "Position", 0, events1[0]),
new StoredEvent(streamName1, "Position", 1, events1[1])
};
var expected2 = new[]
{
new StoredEvent(streamName2, "Position", 0, events2[0]),
new StoredEvent(streamName2, "Position", 1, events2[1])
};
ShouldBeEquivalentTo(readEvents[streamName1], expected1);
ShouldBeEquivalentTo(readEvents[streamName2], expected2);
}
[Theory]
[InlineData(30)]
[InlineData(1000)]
@ -250,7 +289,7 @@ namespace Squidex.Infrastructure.EventSourcing
for (var i = 0; i < count; i++)
{
events.Add(new EventData($"Type{i}", new EnvelopeHeaders(), i.ToString()));
events.Add(CreateEventData(i));
}
for (var i = 0; i < events.Count / 2; i++)
@ -285,8 +324,8 @@ namespace Squidex.Infrastructure.EventSourcing
var events = new[]
{
new EventData("Type1", new EnvelopeHeaders(), "1"),
new EventData("Type2", new EnvelopeHeaders(), "2")
CreateEventData(1),
CreateEventData(2)
};
await Sut.AppendAsync(Guid.NewGuid(), streamName, events);
@ -303,6 +342,11 @@ namespace Squidex.Infrastructure.EventSourcing
return Sut.QueryAsync(streamName, position);
}
private static EventData CreateEventData(int i)
{
return new EventData($"Type{i}", new EnvelopeHeaders(), i.ToString());
}
private async Task<IReadOnlyList<StoredEvent>?> QueryWithCallbackAsync(string? streamFilter = null, string? position = null)
{
using (var cts = new CancellationTokenSource(30000))
@ -370,9 +414,7 @@ namespace Squidex.Infrastructure.EventSourcing
private static void ShouldBeEquivalentTo(IEnumerable<StoredEvent>? actual, params StoredEvent[] expected)
{
var actualArray = actual?.Select(x => new StoredEvent(x.StreamName, "Position", x.EventStreamNumber, x.Data)).ToArray();
actualArray.Should().BeEquivalentTo(expected);
actual.Should().BeEquivalentTo(expected, opts => opts.Excluding(x => x.EventPosition));
}
}
}

164
backend/tests/Squidex.Infrastructure.Tests/States/PersistenceBatchTests.cs

@ -0,0 +1,164 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschränkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FakeItEasy;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.TestHelpers;
using Xunit;
namespace Squidex.Infrastructure.States
{
public class PersistenceBatchTests
{
private readonly ISnapshotStore<int> snapshotStore = A.Fake<ISnapshotStore<int>>();
private readonly IEventDataFormatter eventDataFormatter = A.Fake<IEventDataFormatter>();
private readonly IEventStore eventStore = A.Fake<IEventStore>();
private readonly IStreamNameResolver streamNameResolver = A.Fake<IStreamNameResolver>();
private readonly IStore<int> sut;
public PersistenceBatchTests()
{
A.CallTo(() => streamNameResolver.GetStreamName(None.Type, A<string>._))
.ReturnsLazily(x => x.GetArgument<string>(1)!);
sut = new Store<int>(snapshotStore, eventStore, eventDataFormatter, streamNameResolver);
}
[Fact]
public async Task Should_read_from_preloaded_events()
{
var event1_1 = new MyEvent { MyProperty = "event1_1" };
var event1_2 = new MyEvent { MyProperty = "event1_2" };
var event2_1 = new MyEvent { MyProperty = "event2_1" };
var event2_2 = new MyEvent { MyProperty = "event2_2" };
var key1 = DomainId.NewGuid();
var key2 = DomainId.NewGuid();
var bulk = sut.WithBatchContext(None.Type);
SetupEventStore(new Dictionary<DomainId, List<MyEvent>>
{
[key1] = new List<MyEvent> { event1_1, event1_2 },
[key2] = new List<MyEvent> { event2_1, event2_2 }
});
await bulk.LoadAsync(new[] { key1, key2 });
var persistedEvents1 = Save.Events();
var persistence1 = bulk.WithEventSourcing(None.Type, key1, persistedEvents1.Write);
await persistence1.ReadAsync();
var persistedEvents2 = Save.Events();
var persistence2 = bulk.WithEventSourcing(None.Type, key2, persistedEvents2.Write);
await persistence2.ReadAsync();
Assert.Equal(persistedEvents1.ToArray(), new[] { event1_1, event1_2 });
Assert.Equal(persistedEvents2.ToArray(), new[] { event2_1, event2_2 });
}
[Fact]
public async Task Should_provide_empty_events_if_nothing_loaded()
{
var key = DomainId.NewGuid();
var bulk = sut.WithBatchContext(None.Type);
await bulk.LoadAsync(new[] { key });
var persistedEvents = Save.Events();
var persistence = bulk.WithEventSourcing(None.Type, key, persistedEvents.Write);
await persistence.ReadAsync();
Assert.Empty(persistedEvents.ToArray());
Assert.Empty(persistedEvents.ToArray());
}
[Fact]
public void Should_throw_exception_if_not_preloaded()
{
var key = DomainId.NewGuid();
var bulk = sut.WithBatchContext(None.Type);
Assert.Throws<KeyNotFoundException>(() => bulk.WithEventSourcing(None.Type, key, null));
}
[Fact]
public async Task Should_write_batched()
{
var key1 = DomainId.NewGuid();
var key2 = DomainId.NewGuid();
var bulk = sut.WithBatchContext(None.Type);
await bulk.LoadAsync(new[] { key1, key2 });
var persistedEvents1 = Save.Events();
var persistence1 = bulk.WithEventSourcing(None.Type, key1, persistedEvents1.Write);
var persistedEvents2 = Save.Events();
var persistence2 = bulk.WithEventSourcing(None.Type, key2, persistedEvents2.Write);
await persistence1.WriteSnapshotAsync(12);
await persistence2.WriteSnapshotAsync(12);
A.CallTo(() => snapshotStore.WriteAsync(A<DomainId>._, A<int>._, A<long>._, A<long>._))
.MustNotHaveHappened();
A.CallTo(() => snapshotStore.WriteManyAsync(A<IEnumerable<(DomainId, int, long)>>._))
.MustNotHaveHappened();
await bulk.CommitAsync();
await bulk.DisposeAsync();
A.CallTo(() => snapshotStore.WriteManyAsync(A<IEnumerable<(DomainId, int, long)>>.That.Matches(x => x.Count() == 2)))
.MustHaveHappenedOnceExactly();
}
private void SetupEventStore(Dictionary<DomainId, List<MyEvent>> streams)
{
var storedStreams = new Dictionary<string, IReadOnlyList<StoredEvent>>();
foreach (var (id, stream) in streams)
{
var storedStream = new List<StoredEvent>();
var i = 0;
foreach (var @event in stream)
{
var eventData = new EventData("Type", new EnvelopeHeaders(), "Payload");
var eventStored = new StoredEvent(id.ToString(), i.ToString(), i, eventData);
storedStream.Add(eventStored);
A.CallTo(() => eventDataFormatter.Parse(eventStored))
.Returns(new Envelope<IEvent>(@event));
A.CallTo(() => eventDataFormatter.ParseIfKnown(eventStored))
.Returns(new Envelope<IEvent>(@event));
i++;
}
storedStreams[id.ToString()] = storedStream;
}
var streamNames = streams.Keys.Select(x => x.ToString()).ToArray();
A.CallTo(() => eventStore.QueryManyAsync(A<IEnumerable<string>>.That.IsSameSequenceAs(streamNames)))
.Returns(storedStreams);
}
}
}

51
backend/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs

@ -18,33 +18,26 @@ namespace Squidex.Infrastructure.States
{
public class PersistenceEventSourcingTests
{
private readonly string key = Guid.NewGuid().ToString();
private readonly DomainId key = DomainId.NewGuid();
private readonly ISnapshotStore<int> snapshotStore = A.Fake<ISnapshotStore<int>>();
private readonly IEventDataFormatter eventDataFormatter = A.Fake<IEventDataFormatter>();
private readonly IEventStore eventStore = A.Fake<IEventStore>();
private readonly IServiceProvider services = A.Fake<IServiceProvider>();
private readonly ISnapshotStore<int, string> snapshotStore = A.Fake<ISnapshotStore<int, string>>();
private readonly ISnapshotStore<None, string> snapshotStoreNone = A.Fake<ISnapshotStore<None, string>>();
private readonly IStreamNameResolver streamNameResolver = A.Fake<IStreamNameResolver>();
private readonly IStore<string> sut;
private readonly IStore<int> sut;
public PersistenceEventSourcingTests()
{
A.CallTo(() => services.GetService(typeof(ISnapshotStore<int, string>)))
.Returns(snapshotStore);
A.CallTo(() => services.GetService(typeof(ISnapshotStore<None, string>)))
.Returns(snapshotStoreNone);
A.CallTo(() => streamNameResolver.GetStreamName(None.Type, A<string>._))
.ReturnsLazily(x => x.GetArgument<string>(1)!);
A.CallTo(() => streamNameResolver.GetStreamName(None.Type, key))
.Returns(key);
sut = new Store<string>(eventStore, eventDataFormatter, services, streamNameResolver);
sut = new Store<int>(snapshotStore, eventStore, eventDataFormatter, streamNameResolver);
}
[Fact]
public async Task Should_read_from_store()
{
var event1 = new MyEvent();
var event2 = new MyEvent();
var event1 = new MyEvent { MyProperty = "event1" };
var event2 = new MyEvent { MyProperty = "event2" };
SetupEventStore(event1, event2);
@ -59,8 +52,8 @@ namespace Squidex.Infrastructure.States
[Fact]
public async Task Should_read_until_stopped()
{
var event1 = new MyEvent();
var event2 = new MyEvent();
var event1 = new MyEvent { MyProperty = "event1" };
var event2 = new MyEvent { MyProperty = "event2" };
SetupEventStore(event1, event2);
@ -77,7 +70,7 @@ namespace Squidex.Infrastructure.States
{
var storedEvent = new StoredEvent("1", "1", 0, new EventData("Type", new EnvelopeHeaders(), "Payload"));
A.CallTo(() => eventStore.QueryAsync(key, 0))
A.CallTo(() => eventStore.QueryAsync(key.ToString(), 0))
.Returns(new List<StoredEvent> { storedEvent });
A.CallTo(() => eventDataFormatter.ParseIfKnown(storedEvent))
@ -106,7 +99,7 @@ namespace Squidex.Infrastructure.States
await persistence.ReadAsync();
A.CallTo(() => eventStore.QueryAsync(key, 3))
A.CallTo(() => eventStore.QueryAsync(key.ToString(), 3))
.MustHaveHappened();
}
@ -202,12 +195,12 @@ namespace Squidex.Infrastructure.States
await persistence.WriteEventAsync(Envelope.Create(new MyEvent()));
await persistence.WriteEventAsync(Envelope.Create(new MyEvent()));
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key, 2, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key.ToString(), 2, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
.MustHaveHappened();
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key, 3, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key.ToString(), 3, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
.MustHaveHappened();
A.CallTo(() => snapshotStore.WriteAsync(A<string>._, A<int>._, A<long>._, A<long>._))
A.CallTo(() => snapshotStore.WriteAsync(A<DomainId>._, A<int>._, A<long>._, A<long>._))
.MustNotHaveHappened();
}
@ -218,7 +211,7 @@ namespace Squidex.Infrastructure.States
await persistence.WriteEventAsync(Envelope.Create(new MyEvent()));
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key, EtagVersion.Empty, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key.ToString(), EtagVersion.Empty, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
.MustHaveHappened();
}
@ -304,7 +297,7 @@ namespace Squidex.Infrastructure.States
await persistence.ReadAsync();
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key, 2, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
A.CallTo(() => eventStore.AppendAsync(A<Guid>._, key.ToString(), 2, A<ICollection<EventData>>.That.Matches(x => x.Count == 1)))
.Throws(new WrongEventVersionException(1, 1));
await Assert.ThrowsAsync<InconsistentStateException>(() => persistence.WriteEventAsync(Envelope.Create(new MyEvent())));
@ -317,7 +310,7 @@ namespace Squidex.Infrastructure.States
await persistence.DeleteAsync();
A.CallTo(() => eventStore.DeleteStreamAsync(key))
A.CallTo(() => eventStore.DeleteStreamAsync(key.ToString()))
.MustHaveHappened();
A.CallTo(() => snapshotStore.RemoveAsync(key))
@ -327,11 +320,11 @@ namespace Squidex.Infrastructure.States
[Fact]
public async Task Should_delete_events_and_snapshot_when_deleted()
{
var persistence = sut.WithSnapshotsAndEventSourcing<int>(None.Type, key, null, null);
var persistence = sut.WithSnapshotsAndEventSourcing(None.Type, key, null, null);
await persistence.DeleteAsync();
A.CallTo(() => eventStore.DeleteStreamAsync(key))
A.CallTo(() => eventStore.DeleteStreamAsync(key.ToString()))
.MustHaveHappened();
A.CallTo(() => snapshotStore.RemoveAsync(key))
@ -357,7 +350,7 @@ namespace Squidex.Infrastructure.States
foreach (var @event in events)
{
var eventData = new EventData("Type", new EnvelopeHeaders(), "Payload");
var eventStored = new StoredEvent(i.ToString(), i.ToString(), i, eventData);
var eventStored = new StoredEvent(key.ToString(), i.ToString(), i, eventData);
eventsStored.Add(eventStored);
@ -370,7 +363,7 @@ namespace Squidex.Infrastructure.States
i++;
}
A.CallTo(() => eventStore.QueryAsync(key, readPosition))
A.CallTo(() => eventStore.QueryAsync(key.ToString(), readPosition))
.Returns(eventsStored);
}
}

39
backend/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs

@ -15,20 +15,16 @@ namespace Squidex.Infrastructure.States
{
public class PersistenceSnapshotTests
{
private readonly string key = Guid.NewGuid().ToString();
private readonly DomainId key = DomainId.NewGuid();
private readonly ISnapshotStore<int> snapshotStore = A.Fake<ISnapshotStore<int>>();
private readonly IEventDataFormatter eventDataFormatter = A.Fake<IEventDataFormatter>();
private readonly IEventStore eventStore = A.Fake<IEventStore>();
private readonly IServiceProvider services = A.Fake<IServiceProvider>();
private readonly ISnapshotStore<int, string> snapshotStore = A.Fake<ISnapshotStore<int, string>>();
private readonly IStreamNameResolver streamNameResolver = A.Fake<IStreamNameResolver>();
private readonly IStore<string> sut;
private readonly IStore<int> sut;
public PersistenceSnapshotTests()
{
A.CallTo(() => services.GetService(typeof(ISnapshotStore<int, string>)))
.Returns(snapshotStore);
sut = new Store<string>(eventStore, eventDataFormatter, services, streamNameResolver);
sut = new Store<int>(snapshotStore, eventStore, eventDataFormatter, streamNameResolver);
}
[Fact]
@ -122,7 +118,7 @@ namespace Squidex.Infrastructure.States
[Fact]
public async Task Should_write_snapshot_to_store_with_empty_version()
{
var persistence = sut.WithSnapshots<int>(None.Type, key, null);
var persistence = sut.WithSnapshots(None.Type, key, null);
await persistence.WriteSnapshotAsync(100);
@ -165,33 +161,10 @@ namespace Squidex.Infrastructure.States
[Fact]
public async Task Should_call_snapshot_store_on_clear()
{
await sut.ClearSnapshotsAsync<string, int>();
await sut.ClearSnapshotsAsync();
A.CallTo(() => snapshotStore.ClearAsync())
.MustHaveHappened();
}
[Fact]
public async Task Should_delete_snapshot_but_not_events_when_deleted_from_store()
{
await sut.RemoveSnapshotAsync<string, int>(key);
A.CallTo(() => eventStore.DeleteStreamAsync(A<string>._))
.MustNotHaveHappened();
A.CallTo(() => snapshotStore.RemoveAsync(key))
.MustHaveHappened();
}
[Fact]
public async Task Should_get_snapshot()
{
A.CallTo(() => snapshotStore.ReadAsync(key))
.Returns((123, -1));
var result = await sut.GetSnapshotAsync<string, int>(key);
Assert.Equal(123, result);
}
}
}

2
backend/tests/Squidex.Infrastructure.Tests/Tasks/PartitionedActionBlockTests.cs

@ -39,7 +39,7 @@ namespace Squidex.Infrastructure.Tasks
}, x => x.P, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 100,
MaxMessagesPerTask = 1,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
BoundedCapacity = 100
});

4
backend/tests/Squidex.Infrastructure.Tests/TestHelpers/MyDomainObject.cs

@ -24,8 +24,8 @@ namespace Squidex.Infrastructure.TestHelpers
set => Capacity = value;
}
public MyDomainObject(IStore<DomainId> store)
: base(store, A.Dummy<ISemanticLog>())
public MyDomainObject(IPersistenceFactory<MyDomainState> factory)
: base(factory, A.Dummy<ISemanticLog>())
{
}

Loading…
Cancel
Save