Browse Source

Faster migration.

pull/349/head
Sebastian 7 years ago
parent
commit
9fc698f79b
  1. 2
      src/Squidex.Domain.Apps.Core.Model/SquidexCoreModel.cs
  2. 2
      src/Squidex.Domain.Apps.Core.Operations/SquidexCoreOperations.cs
  3. 7
      src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs
  4. 13
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs
  5. 31
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs
  6. 5
      src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs
  7. 2
      src/Squidex.Domain.Apps.Entities/Apps/State/AppState.cs
  8. 5
      src/Squidex.Domain.Apps.Entities/Assets/AssetGrain.cs
  9. 4
      src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs
  10. 2
      src/Squidex.Domain.Apps.Entities/Assets/State/AssetState.cs
  11. 5
      src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs
  12. 6
      src/Squidex.Domain.Apps.Entities/Comments/State/CommentsState.cs
  13. 2
      src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs
  14. 5
      src/Squidex.Domain.Apps.Entities/Contents/ContentGrain.cs
  15. 2
      src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs
  16. 1
      src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs
  17. 5
      src/Squidex.Domain.Apps.Entities/DomainObjectState.cs
  18. 5
      src/Squidex.Domain.Apps.Entities/Rules/RuleGrain.cs
  19. 2
      src/Squidex.Domain.Apps.Entities/Rules/State/RuleState.cs
  20. 5
      src/Squidex.Domain.Apps.Entities/Schemas/SchemaGrain.cs
  21. 2
      src/Squidex.Domain.Apps.Entities/Schemas/State/SchemaState.cs
  22. 2
      src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs
  23. 2
      src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrainLogSnapshots.cs
  24. 2
      src/Squidex.Domain.Apps.Entities/SquidexEntities.cs
  25. 2
      src/Squidex.Domain.Apps.Events/SquidexEvents.cs
  26. 10
      src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs
  27. 5
      src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs
  28. 7
      src/Squidex.Infrastructure/Commands/DomainObjectGrain.cs
  29. 2
      src/Squidex.Infrastructure/Commands/DomainObjectGrainBase.cs
  30. 6
      src/Squidex.Infrastructure/Commands/IDomainState.cs
  31. 7
      src/Squidex.Infrastructure/Commands/LogSnapshotDomainObjectGrain.cs
  32. 2
      src/Squidex.Infrastructure/SquidexInfrastructure.cs
  33. 3
      src/Squidex.Infrastructure/States/ISnapshotStore.cs
  34. 3
      src/Squidex/Config/Domain/EntitiesServices.cs
  35. 6
      src/Squidex/Config/Domain/StoreServices.cs
  36. 5
      tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs
  37. 37
      tests/Squidex.Infrastructure.Tests/Commands/DomainObjectGrainTests.cs
  38. 40
      tests/Squidex.Infrastructure.Tests/Commands/LogSnapshotDomainObjectGrainTests.cs
  39. 45
      tests/Squidex.Infrastructure.Tests/TestHelpers/MyDomainObject.cs
  40. 13
      tests/Squidex.Infrastructure.Tests/TestHelpers/MyDomainState.cs
  41. 6
      tests/Squidex.Infrastructure.Tests/TestHelpers/MyGrain.cs
  42. 11
      tools/Migrate_01/MigrationPath.cs
  43. 50
      tools/Migrate_01/Migrations/BuildFullTextIndices.cs
  44. 2
      tools/Migrate_01/Migrations/MongoDb/ConvertOldSnapshotStores.cs
  45. 2
      tools/Migrate_01/Migrations/MongoDb/ConvertRuleEventsJson.cs
  46. 2
      tools/Migrate_01/Migrations/MongoDb/DeleteContentCollections.cs
  47. 39
      tools/Migrate_01/Migrations/MongoDb/RestructureContentCollection.cs
  48. 90
      tools/Migrate_01/Rebuilder.cs
  49. 2
      tools/Migrate_01/SquidexMigrations.cs

2
src/Squidex.Domain.Apps.Core.Model/SquidexCoreModel.cs

@ -7,6 +7,8 @@
using System.Reflection;
#pragma warning disable RECS0014 // If all fields, properties and methods members are static, the class can be made static.
namespace Squidex.Domain.Apps.Core
{
public sealed class SquidexCoreModel

2
src/Squidex.Domain.Apps.Core.Operations/SquidexCoreOperations.cs

@ -7,6 +7,8 @@
using System.Reflection;
#pragma warning disable RECS0014 // If all fields, properties and methods members are static, the class can be made static.
namespace Squidex.Domain.Apps.Core
{
public static class SquidexCoreOperations

7
src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs

@ -6,6 +6,7 @@
// ==========================================================================
using System;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using Squidex.Domain.Apps.Entities.Assets.State;
@ -18,7 +19,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
{
public sealed partial class MongoAssetRepository : ISnapshotStore<AssetState, Guid>
{
public async Task<(AssetState Value, long Version)> ReadAsync(Guid key)
async Task<(AssetState Value, long Version)> ISnapshotStore<AssetState, Guid>.ReadAsync(Guid key)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
{
@ -35,7 +36,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
public async Task WriteAsync(Guid key, AssetState value, long oldVersion, long newVersion)
async Task ISnapshotStore<AssetState, Guid>.WriteAsync(Guid key, AssetState value, long oldVersion, long newVersion)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
{
@ -48,7 +49,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
}
}
Task ISnapshotStore<AssetState, Guid>.ReadAllAsync(Func<AssetState, long, Task> callback)
Task ISnapshotStore<AssetState, Guid>.ReadAllAsync(Func<AssetState, long, Task> callback, CancellationToken ct)
{
throw new NotSupportedException();
}

13
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs

@ -10,6 +10,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using NodaTime;
using Squidex.Domain.Apps.Core.Contents;
@ -170,6 +171,18 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
return (null, EtagVersion.NotFound);
}
public Task ReadAllAsync(Func<ContentState, long, Task> callback, Func<Guid, Guid, Task<ISchemaEntity>> getSchema, CancellationToken ct = default)
{
return Collection.Find(new BsonDocument()).ForEachPipelineAsync(async contentEntity =>
{
var schema = await getSchema(contentEntity.IndexedAppId, contentEntity.IndexedSchemaId);
contentEntity.ParseData(schema.SchemaDef, Serializer);
await callback(SimpleMapper.Map(contentEntity, new ContentState()), contentEntity.Version);
}, ct);
}
public Task CleanupAsync(Guid id)
{
return Collection.UpdateManyAsync(

31
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs

@ -6,6 +6,7 @@
// ==========================================================================
using System;
using System.Threading;
using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Contents.State;
using Squidex.Domain.Apps.Entities.Schemas;
@ -18,7 +19,23 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
{
public partial class MongoContentRepository : ISnapshotStore<ContentState, Guid>
{
public async Task<(ContentState Value, long Version)> ReadAsync(Guid key)
async Task ISnapshotStore<ContentState, Guid>.RemoveAsync(Guid key)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
await contents.RemoveAsync(key);
}
}
async Task ISnapshotStore<ContentState, Guid>.ReadAllAsync(Func<ContentState, long, Task> callback, CancellationToken ct)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
await contents.ReadAllAsync(callback, GetSchemaAsync, ct);
}
}
async Task<(ContentState Value, long Version)> ISnapshotStore<ContentState, Guid>.ReadAsync(Guid key)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
@ -26,7 +43,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
}
}
public async Task WriteAsync(Guid key, ContentState value, long oldVersion, long newVersion)
async Task ISnapshotStore<ContentState, Guid>.WriteAsync(Guid key, ContentState value, long oldVersion, long newVersion)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
@ -81,15 +98,5 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
return schema;
}
Task ISnapshotStore<ContentState, Guid>.RemoveAsync(Guid key)
{
throw new NotSupportedException();
}
Task ISnapshotStore<ContentState, Guid>.ReadAllAsync(Func<ContentState, long, Task> callback)
{
throw new NotSupportedException();
}
}
}

5
src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs

@ -369,11 +369,6 @@ namespace Squidex.Domain.Apps.Entities.Apps
return new AppContributorAssigned { ContributorId = actor.Identifier, Role = Role.Owner };
}
protected override AppState OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
public Task<J<IAppEntity>> GetStateAsync()
{
return J.AsTask<IAppEntity>(Snapshot);

2
src/Squidex.Domain.Apps.Entities/Apps/State/AppState.cs

@ -142,7 +142,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.State
IsArchived = true;
}
public AppState Apply(Envelope<IEvent> @event)
public override AppState Apply(Envelope<IEvent> @event)
{
var payload = (SquidexEvent)@event.Payload;

5
src/Squidex.Domain.Apps.Entities/Assets/AssetGrain.cs

@ -165,11 +165,6 @@ namespace Squidex.Domain.Apps.Entities.Assets
}
}
protected override AssetState OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
public Task<J<IAssetEntity>> GetStateAsync(long version = EtagVersion.Any)
{
return J.AsTask<IAssetEntity>(GetSnapshot(version));

4
src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs

@ -77,7 +77,7 @@ namespace Squidex.Domain.Apps.Entities.Assets
{
await RestoreTagsAsync(appId, reader);
await RebuildManyAsync(assetIds, id => RebuildAsync<AssetState, AssetGrain>(id, (e, s) => s.Apply(e)));
await RebuildManyAsync(assetIds, id => RebuildAsync<AssetState, AssetGrain>(id));
}
private async Task RestoreTagsAsync(Guid appId, BackupReader reader)
@ -110,7 +110,7 @@ namespace Squidex.Domain.Apps.Entities.Assets
{
try
{
await assetStore.UploadAsync(assetId.ToString(), fileVersion, null, stream);
await assetStore.UploadAsync(assetId.ToString(), fileVersion, null, stream, true);
}
catch (AssetAlreadyExistsException)
{

2
src/Squidex.Domain.Apps.Entities/Assets/State/AssetState.cs

@ -89,7 +89,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.State
IsDeleted = true;
}
public AssetState Apply(Envelope<IEvent> @event)
public override AssetState Apply(Envelope<IEvent> @event)
{
var payload = (SquidexEvent)@event.Payload;

5
src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs

@ -10,7 +10,6 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
namespace Squidex.Domain.Apps.Entities.Backup
@ -39,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
}
}
protected async Task RebuildAsync<TState, TGrain>(Guid key, Func<Envelope<IEvent>, TState, TState> func) where TState : IDomainState, new()
protected async Task RebuildAsync<TState, TGrain>(Guid key) where TState : IDomainState<TState>, new()
{
var state = new TState
{
@ -48,7 +47,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
var persistence = store.WithSnapshotsAndEventSourcing(typeof(TGrain), key, (TState s) => state = s, e =>
{
state = func(e, state);
state = state.Apply(e);
state.Version++;
});

6
src/Squidex.Domain.Apps.Entities/Comments/State/CommentsState.cs

@ -5,9 +5,15 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Domain.Apps.Entities.Comments.State
{
public sealed class CommentsState : DomainObjectState<CommentsState>
{
public override CommentsState Apply(Envelope<IEvent> @event)
{
return this;
}
}
}

2
src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs

@ -50,7 +50,7 @@ namespace Squidex.Domain.Apps.Entities.Contents
{
var contentIds = contentIdsBySchemaId.Values.SelectMany(x => x);
return RebuildManyAsync(contentIds, id => RebuildAsync<ContentState, ContentGrain>(id, (e, s) => s.Apply(e)));
return RebuildManyAsync(contentIds, id => RebuildAsync<ContentState, ContentGrain>(id));
}
}
}

5
src/Squidex.Domain.Apps.Entities/Contents/ContentGrain.cs

@ -296,11 +296,6 @@ namespace Squidex.Domain.Apps.Entities.Contents
}
}
protected override ContentState OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
private async Task<ContentOperationContext> CreateContext(Guid appId, Guid schemaId, Guid contentId, Func<string> message)
{
var operationContext =

2
src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs

@ -111,7 +111,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.State
IsDeleted = true;
}
public ContentState Apply(Envelope<IEvent> @event)
public override ContentState Apply(Envelope<IEvent> @event)
{
var payload = (SquidexEvent)@event.Payload;

1
src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs

@ -12,7 +12,6 @@ using System.Threading.Tasks;
using Orleans;
using Squidex.Domain.Apps.Core.Contents;
using Squidex.Domain.Apps.Entities.Apps;
using Squidex.Domain.Apps.Entities.Schemas;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Log;

5
src/Squidex.Domain.Apps.Entities/DomainObjectState.cs

@ -10,11 +10,12 @@ using System.Runtime.Serialization;
using NodaTime;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Domain.Apps.Entities
{
public abstract class DomainObjectState<T> : Cloneable<T>,
IDomainState,
IDomainState<T>,
IEntity,
IEntityWithCreatedBy,
IEntityWithLastModifiedBy,
@ -42,6 +43,8 @@ namespace Squidex.Domain.Apps.Entities
[DataMember]
public long Version { get; set; } = EtagVersion.Empty;
public abstract T Apply(Envelope<IEvent> @event);
public T Clone()
{
return Clone(x => { });

5
src/Squidex.Domain.Apps.Entities/Rules/RuleGrain.cs

@ -123,11 +123,6 @@ namespace Squidex.Domain.Apps.Entities.Rules
}
}
protected override RuleState OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
public Task<J<IRuleEntity>> GetStateAsync()
{
return J.AsTask<IRuleEntity>(Snapshot);

2
src/Squidex.Domain.Apps.Entities/Rules/State/RuleState.cs

@ -64,7 +64,7 @@ namespace Squidex.Domain.Apps.Entities.Rules.State
IsDeleted = true;
}
public RuleState Apply(Envelope<IEvent> @event)
public override RuleState Apply(Envelope<IEvent> @event)
{
var payload = (SquidexEvent)@event.Payload;

5
src/Squidex.Domain.Apps.Entities/Schemas/SchemaGrain.cs

@ -381,11 +381,6 @@ namespace Squidex.Domain.Apps.Entities.Schemas
}
}
protected override SchemaState OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
public Task<J<ISchemaEntity>> GetStateAsync()
{
return J.AsTask<ISchemaEntity>(Snapshot);

2
src/Squidex.Domain.Apps.Entities/Schemas/State/SchemaState.cs

@ -137,7 +137,7 @@ namespace Squidex.Domain.Apps.Entities.Schemas.State
IsDeleted = true;
}
public SchemaState Apply(Envelope<IEvent> @event)
public override SchemaState Apply(Envelope<IEvent> @event)
{
var payload = (SquidexEvent)@event.Payload;

2
src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs

@ -14,7 +14,7 @@ using Squidex.Infrastructure.States;
namespace Squidex.Domain.Apps.Entities
{
public abstract class SquidexDomainObjectGrain<T> : DomainObjectGrain<T> where T : IDomainState, new()
public abstract class SquidexDomainObjectGrain<T> : DomainObjectGrain<T> where T : IDomainState<T>, new()
{
protected SquidexDomainObjectGrain(IStore<Guid> store, ISemanticLog log)
: base(store, log)

2
src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrainLogSnapshots.cs

@ -14,7 +14,7 @@ using Squidex.Infrastructure.States;
namespace Squidex.Domain.Apps.Entities
{
public abstract class SquidexDomainObjectGrainLogSnapshots<T> : LogSnapshotDomainObjectGrain<T> where T : IDomainState, new()
public abstract class SquidexDomainObjectGrainLogSnapshots<T> : LogSnapshotDomainObjectGrain<T> where T : IDomainState<T>, new()
{
protected SquidexDomainObjectGrainLogSnapshots(IStore<Guid> store, ISemanticLog log)
: base(store, log)

2
src/Squidex.Domain.Apps.Entities/SquidexEntities.cs

@ -7,6 +7,8 @@
using System.Reflection;
#pragma warning disable RECS0014 // If all fields, properties and methods members are static, the class can be made static.
namespace Squidex.Domain.Apps.Entities
{
public static class SquidexEntities

2
src/Squidex.Domain.Apps.Events/SquidexEvents.cs

@ -7,6 +7,8 @@
using System.Reflection;
#pragma warning disable RECS0014 // If all fields, properties and methods members are static, the class can be made static.
namespace Squidex.Domain.Apps.Events
{
public sealed class SquidexEvents

10
src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs

@ -22,6 +22,16 @@ namespace Squidex.Infrastructure.MongoDb
{
private static readonly UpdateOptions Upsert = new UpdateOptions { IsUpsert = true };
public static async Task<bool> CollectionExistsAsync(this IMongoDatabase database, string collectionName)
{
var options = new ListCollectionNamesOptions
{
Filter = new BsonDocument("name", collectionName)
};
return (await database.ListCollectionNamesAsync(options)).Any();
}
public static async Task<bool> InsertOneIfNotExistsAsync<T>(this IMongoCollection<T> collection, T document)
{
try

5
src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs

@ -7,6 +7,7 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
@ -60,11 +61,11 @@ namespace Squidex.Infrastructure.States
}
}
public async Task ReadAllAsync(Func<T, long, Task> callback)
public async Task ReadAllAsync(Func<T, long, Task> callback, CancellationToken ct = default)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
{
await Collection.Find(new BsonDocument()).ForEachAsync(x => callback(x.Doc, x.Version));
await Collection.Find(new BsonDocument()).ForEachPipelineAsync(x => callback(x.Doc, x.Version), ct);
}
}

7
src/Squidex.Infrastructure/Commands/DomainObjectGrain.cs

@ -13,7 +13,7 @@ using Squidex.Infrastructure.States;
namespace Squidex.Infrastructure.Commands
{
public abstract class DomainObjectGrain<T> : DomainObjectGrainBase<T> where T : IDomainState, new()
public abstract class DomainObjectGrain<T> : DomainObjectGrainBase<T> where T : IDomainState<T>, new()
{
private readonly IStore<Guid> store;
private T snapshot = new T { Version = EtagVersion.Empty };
@ -66,6 +66,9 @@ namespace Squidex.Infrastructure.Commands
}
}
protected abstract T OnEvent(Envelope<IEvent> @event);
protected T OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
}
}

2
src/Squidex.Infrastructure/Commands/DomainObjectGrainBase.cs

@ -15,7 +15,7 @@ using Squidex.Infrastructure.Tasks;
namespace Squidex.Infrastructure.Commands
{
public abstract class DomainObjectGrainBase<T> : GrainOfGuid, IDomainObjectGrain where T : IDomainState, new()
public abstract class DomainObjectGrainBase<T> : GrainOfGuid, IDomainObjectGrain where T : IDomainState<T>, new()
{
private readonly List<Envelope<IEvent>> uncomittedEvents = new List<Envelope<IEvent>>();
private readonly ISemanticLog log;

6
src/Squidex.Infrastructure/Commands/IDomainState.cs

@ -5,10 +5,14 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.Commands
{
public interface IDomainState
public interface IDomainState<T>
{
long Version { get; set; }
T Apply(Envelope<IEvent> @event);
}
}

7
src/Squidex.Infrastructure/Commands/LogSnapshotDomainObjectGrain.cs

@ -15,7 +15,7 @@ using Squidex.Infrastructure.States;
namespace Squidex.Infrastructure.Commands
{
public abstract class LogSnapshotDomainObjectGrain<T> : DomainObjectGrainBase<T> where T : IDomainState, new()
public abstract class LogSnapshotDomainObjectGrain<T> : DomainObjectGrainBase<T> where T : IDomainState<T>, new()
{
private readonly IStore<Guid> store;
private readonly List<T> snapshots = new List<T> { new T { Version = EtagVersion.Empty } };
@ -88,6 +88,9 @@ namespace Squidex.Infrastructure.Commands
}
}
protected abstract T OnEvent(Envelope<IEvent> @event);
protected T OnEvent(Envelope<IEvent> @event)
{
return Snapshot.Apply(@event);
}
}
}

2
src/Squidex.Infrastructure/SquidexInfrastructure.cs

@ -7,6 +7,8 @@
using System.Reflection;
#pragma warning disable RECS0014 // If all fields, properties and methods members are static, the class can be made static.
namespace Squidex.Infrastructure
{
public sealed class SquidexInfrastructure

3
src/Squidex.Infrastructure/States/ISnapshotStore.cs

@ -6,6 +6,7 @@
// ==========================================================================
using System;
using System.Threading;
using System.Threading.Tasks;
namespace Squidex.Infrastructure.States
@ -20,6 +21,6 @@ namespace Squidex.Infrastructure.States
Task RemoveAsync(TKey key);
Task ReadAllAsync(Func<T, long, Task> callback);
Task ReadAllAsync(Func<T, long, Task> callback, CancellationToken ct = default);
}
}

3
src/Squidex/Config/Domain/EntitiesServices.cs

@ -249,6 +249,9 @@ namespace Squidex.Config.Domain
services.AddTransientAs<AddPatterns>()
.As<IMigration>();
services.AddTransientAs<BuildFullTextIndices>()
.As<IMigration>();
services.AddTransientAs<ConvertEventStore>()
.As<IMigration>();

6
src/Squidex/Config/Domain/StoreServices.cs

@ -10,8 +10,7 @@ using IdentityServer4.Stores;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using Migrate_01.Migrations;
using Migrate_01.Migrations.MongoDb;
using MongoDB.Driver;
using Squidex.Domain.Apps.Entities;
using Squidex.Domain.Apps.Entities.Assets.Repositories;
@ -66,6 +65,9 @@ namespace Squidex.Config.Domain
services.AddTransientAs(c => new DeleteContentCollections(c.GetRequiredService<IMongoClient>().GetDatabase(mongoContentDatabaseName)))
.As<IMigration>();
services.AddTransientAs(c => new RestructureContentCollection(c.GetRequiredService<IMongoClient>().GetDatabase(mongoContentDatabaseName)))
.As<IMigration>();
services.AddSingletonAs<MongoMigrationStatus>()
.As<IMigrationStatus>();

5
tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs

@ -7,6 +7,7 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using FakeItEasy;
@ -38,8 +39,8 @@ namespace Squidex.Domain.Users
[Fact]
public void Should_return_items_from_store()
{
A.CallTo(() => store.ReadAllAsync(A<Func<DefaultXmlRepository.State, long, Task>>.Ignored))
.Invokes((Func<DefaultXmlRepository.State, long, Task> callback) =>
A.CallTo(() => store.ReadAllAsync(A<Func<DefaultXmlRepository.State, long, Task>>.Ignored, A<CancellationToken>.Ignored))
.Invokes((Func<DefaultXmlRepository.State, long, Task> callback, CancellationToken ct) =>
{
callback(new DefaultXmlRepository.State { Xml = "<a />" }, EtagVersion.Any);
callback(new DefaultXmlRepository.State { Xml = "<b />" }, EtagVersion.Any);

37
tests/Squidex.Infrastructure.Tests/Commands/DomainObjectGrainTests.cs

@ -26,38 +26,6 @@ namespace Squidex.Infrastructure.Commands
private readonly Guid id = Guid.NewGuid();
private readonly MyDomainObject sut;
public sealed class MyDomainState : IDomainState
{
public long Version { get; set; }
public int Value { get; set; }
}
public sealed class ValueChanged : IEvent
{
public int Value { get; set; }
}
public sealed class CreateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class CreateCustom : MyCommand
{
public int Value { get; set; }
}
public sealed class UpdateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class UpdateCustom : MyCommand
{
public int Value { get; set; }
}
public sealed class MyDomainObject : DomainObjectGrain<MyDomainState>
{
public MyDomainObject(IStore<Guid> store)
@ -100,11 +68,6 @@ namespace Squidex.Infrastructure.Commands
return Task.FromResult<object>(null);
}
protected override MyDomainState OnEvent(Envelope<IEvent> @event)
{
return new MyDomainState { Value = ((ValueChanged)@event.Payload).Value };
}
}
public DomainObjectGrainTests()

40
tests/Squidex.Infrastructure.Tests/Commands/LogSnapshotDomainObjectGrainTests.cs

@ -26,36 +26,11 @@ namespace Squidex.Infrastructure.Commands
private readonly ISnapshotStore<MyDomainState, Guid> snapshotStore = A.Fake<ISnapshotStore<MyDomainState, Guid>>();
private readonly IPersistence persistence = A.Fake<IPersistence>();
private readonly Guid id = Guid.NewGuid();
private readonly MyDomainObject sut;
private readonly MyLogDomainObject sut;
public sealed class ValueChanged : IEvent
public sealed class MyLogDomainObject : LogSnapshotDomainObjectGrain<MyDomainState>
{
public int Value { get; set; }
}
public sealed class CreateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class CreateCustom : MyCommand
{
public int Value { get; set; }
}
public sealed class UpdateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class UpdateCustom : MyCommand
{
public int Value { get; set; }
}
public sealed class MyDomainObject : LogSnapshotDomainObjectGrain<MyDomainState>
{
public MyDomainObject(IStore<Guid> store)
public MyLogDomainObject(IStore<Guid> store)
: base(store, A.Dummy<ISemanticLog>())
{
}
@ -95,22 +70,17 @@ namespace Squidex.Infrastructure.Commands
return Task.FromResult<object>(null);
}
protected override MyDomainState OnEvent(Envelope<IEvent> @event)
{
return new MyDomainState { Value = ((ValueChanged)@event.Payload).Value };
}
}
public LogSnapshotDomainObjectGrainTests()
{
A.CallTo(() => store.WithEventSourcing(typeof(MyDomainObject), id, A<HandleEvent>.Ignored))
A.CallTo(() => store.WithEventSourcing(typeof(MyLogDomainObject), id, A<HandleEvent>.Ignored))
.Returns(persistence);
A.CallTo(() => store.GetSnapshotStore<MyDomainState>())
.Returns(snapshotStore);
sut = new MyDomainObject(store);
sut = new MyLogDomainObject(store);
}
[Fact]

45
tests/Squidex.Infrastructure.Tests/TestHelpers/MyDomainObject.cs

@ -9,7 +9,6 @@ using System;
using System.Threading.Tasks;
using FakeItEasy;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Log;
using Squidex.Infrastructure.States;
@ -17,31 +16,6 @@ namespace Squidex.Infrastructure.TestHelpers
{
public sealed class MyDomainObject : DomainObjectGrain<MyDomainState>
{
public sealed class ValueChanged : IEvent
{
public int Value { get; set; }
}
public sealed class CreateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class CreateCustom : MyCommand
{
public int Value { get; set; }
}
public sealed class UpdateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class UpdateCustom : MyCommand
{
public int Value { get; set; }
}
public MyDomainObject(IStore<Guid> store)
: base(store, A.Dummy<ISemanticLog>())
{
@ -82,10 +56,25 @@ namespace Squidex.Infrastructure.TestHelpers
return Task.FromResult<object>(null);
}
}
public sealed class CreateAuto : MyCommand
{
public int Value { get; set; }
}
public sealed class CreateCustom : MyCommand
{
public int Value { get; set; }
}
protected override MyDomainState OnEvent(Envelope<IEvent> @event)
public sealed class UpdateAuto : MyCommand
{
return new MyDomainState { Value = ((ValueChanged)@event.Payload).Value };
public int Value { get; set; }
}
public sealed class UpdateCustom : MyCommand
{
public int Value { get; set; }
}
}

13
tests/Squidex.Infrastructure.Tests/TestHelpers/MyDomainState.cs

@ -6,13 +6,24 @@
// ==========================================================================
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Infrastructure.TestHelpers
{
public sealed class MyDomainState : IDomainState
public sealed class MyDomainState : IDomainState<MyDomainState>
{
public long Version { get; set; }
public int Value { get; set; }
public MyDomainState Apply(Envelope<IEvent> @event)
{
return new MyDomainState { Value = ((ValueChanged)@event.Payload).Value };
}
}
public sealed class ValueChanged : IEvent
{
public int Value { get; set; }
}
}

6
tests/Squidex.Infrastructure.Tests/TestHelpers/MyGrain.cs

@ -9,7 +9,6 @@ using System;
using System.Threading.Tasks;
using FakeItEasy;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Log;
using Squidex.Infrastructure.States;
@ -26,10 +25,5 @@ namespace Squidex.Infrastructure.TestHelpers
{
return Task.FromResult<object>(null);
}
protected override MyDomainState OnEvent(Envelope<IEvent> @event)
{
return Snapshot;
}
}
}

11
tools/Migrate_01/MigrationPath.cs

@ -10,6 +10,7 @@ using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Migrate_01.Migrations;
using Migrate_01.Migrations.MongoDb;
using Squidex.Infrastructure.Migrations;
namespace Migrate_01
@ -72,8 +73,7 @@ namespace Migrate_01
}
// Version 11: Introduce content drafts.
// Version 15: Introduce custom full text search actors.
if (version < 15)
if (version < 11)
{
yield return serviceProvider.GetService<DeleteContentCollections>();
yield return serviceProvider.GetRequiredService<RebuildContents>();
@ -97,6 +97,13 @@ namespace Migrate_01
yield return serviceProvider.GetRequiredService<AddPatterns>();
}
// Version 15: Introduce custom full text search actors.
if (version < 15)
{
yield return serviceProvider.GetService<RestructureContentCollection>();
yield return serviceProvider.GetService<BuildFullTextIndices>();
}
yield return serviceProvider.GetRequiredService<StartEventConsumers>();
}
}

50
tools/Migrate_01/Migrations/BuildFullTextIndices.cs

@ -0,0 +1,50 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Squidex.Domain.Apps.Entities.Contents.State;
using Squidex.Domain.Apps.Entities.Contents.Text;
using Squidex.Infrastructure.Migrations;
using Squidex.Infrastructure.States;
namespace Migrate_01.Migrations
{
public sealed class BuildFullTextIndices : IMigration
{
private readonly ITextIndexer textIndexer;
private readonly IStore<Guid> store;
public BuildFullTextIndices(ITextIndexer textIndexer, IStore<Guid> store)
{
this.textIndexer = textIndexer;
this.store = store;
}
public async Task UpdateAsync()
{
var snapshotStore = store.GetSnapshotStore<ContentState>();
var worker = new ActionBlock<ContentState>(state =>
{
return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.Data, state.DataDraft);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
});
await snapshotStore.ReadAllAsync((state, version) => worker.SendAsync(state));
worker.Complete();
await worker.Completion;
}
}
}

2
tools/Migrate_01/Migrations/ConvertOldSnapshotStores.cs → tools/Migrate_01/Migrations/MongoDb/ConvertOldSnapshotStores.cs

@ -14,7 +14,7 @@ using Squidex.Infrastructure.Migrations;
using Squidex.Infrastructure.MongoDb;
using Squidex.Infrastructure.Tasks;
namespace Migrate_01.Migrations
namespace Migrate_01.Migrations.MongoDb
{
public sealed class ConvertOldSnapshotStores : IMigration
{

2
tools/Migrate_01/Migrations/ConvertRuleEventsJson.cs → tools/Migrate_01/Migrations/MongoDb/ConvertRuleEventsJson.cs

@ -10,7 +10,7 @@ using MongoDB.Bson;
using MongoDB.Driver;
using Squidex.Infrastructure.Migrations;
namespace Migrate_01.Migrations
namespace Migrate_01.Migrations.MongoDb
{
public sealed class ConvertRuleEventsJson : IMigration
{

2
tools/Migrate_01/Migrations/DeleteContentCollections.cs → tools/Migrate_01/Migrations/MongoDb/DeleteContentCollections.cs

@ -9,7 +9,7 @@ using System.Threading.Tasks;
using MongoDB.Driver;
using Squidex.Infrastructure.Migrations;
namespace Migrate_01.Migrations
namespace Migrate_01.Migrations.MongoDb
{
public sealed class DeleteContentCollections : IMigration
{

39
tools/Migrate_01/Migrations/MongoDb/RestructureContentCollection.cs

@ -0,0 +1,39 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using Squidex.Infrastructure.Migrations;
using Squidex.Infrastructure.MongoDb;
namespace Migrate_01.Migrations.MongoDb
{
public sealed class RestructureContentCollection : IMigration
{
private readonly IMongoDatabase contentDatabase;
public RestructureContentCollection(IMongoDatabase contentDatabase)
{
this.contentDatabase = contentDatabase;
}
public async Task UpdateAsync()
{
if (await contentDatabase.CollectionExistsAsync("State_Content_Draft"))
{
await contentDatabase.DropCollectionAsync("State_Contents");
await contentDatabase.DropCollectionAsync("State_Content_Published");
await contentDatabase.RenameCollectionAsync("State_Content_Draft", "State_Contents");
var collection = contentDatabase.GetCollection<BsonDocument>("State_Content");
await collection.UpdateManyAsync(new BsonDocument(), Builders<BsonDocument>.Update.Unset("dt"));
}
}
}
}

90
tools/Migrate_01/Rebuilder.cs

@ -7,7 +7,6 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Squidex.Domain.Apps.Entities.Apps;
@ -44,97 +43,82 @@ namespace Migrate_01
this.store = store;
}
public async Task RebuildAppsAsync()
public Task RebuildAppsAsync()
{
await store.GetSnapshotStore<AppState>().ClearAsync();
await RebuildManyAsync("^app\\-", id => RebuildAsync<AppState, AppGrain>(id, (e, s) => s.Apply(e)));
return RebuildManyAsync<AppState, AppGrain>("^app\\-");
}
public async Task RebuildSchemasAsync()
public Task RebuildSchemasAsync()
{
await store.GetSnapshotStore<SchemaState>().ClearAsync();
await RebuildManyAsync("^schema\\-", id => RebuildAsync<SchemaState, SchemaGrain>(id, (e, s) => s.Apply(e)));
return RebuildManyAsync<SchemaState, SchemaGrain>("^schema\\-");
}
public async Task RebuildRulesAsync()
public Task RebuildRulesAsync()
{
await store.GetSnapshotStore<RuleState>().ClearAsync();
await RebuildManyAsync("^rule\\-", id => RebuildAsync<RuleState, RuleGrain>(id, (e, s) => s.Apply(e)));
return RebuildManyAsync<RuleState, RuleGrain>("^rule\\-");
}
public async Task RebuildAssetsAsync()
public Task RebuildAssetsAsync()
{
await store.GetSnapshotStore<AssetState>().ClearAsync();
await RebuildManyAsync("^asset\\-", id => RebuildAsync<AssetState, AssetGrain>(id, (e, s) => s.Apply(e)));
return RebuildManyAsync<AssetState, AssetGrain>("^asset\\-");
}
public async Task RebuildContentAsync()
public Task RebuildContentAsync()
{
using (localCache.StartContext())
return RebuildManyAsync<ContentState, ContentGrain>("^content\\-");
}
private async Task RebuildManyAsync<TState, TGrain>(string filter) where TState : IDomainState<TState>, new()
{
await store.GetSnapshotStore<ContentState>().ClearAsync();
var handledIds = new HashSet<Guid>();
await RebuildManyAsync("^content\\-", async id =>
var worker = new ActionBlock<Guid>(async id =>
{
try
{
await RebuildAsync<ContentState, ContentGrain>(id, (e, s) => s.Apply(e));
var state = new TState
{
Version = EtagVersion.Empty
};
var persistence = store.WithSnapshotsAndEventSourcing(typeof(TGrain), id, (TState s) => state = s, e =>
{
state = state.Apply(e);
state.Version++;
});
await persistence.ReadAsync();
await persistence.WriteSnapshotAsync(state);
}
catch (DomainObjectNotFoundException)
{
return;
}
});
}
}
private async Task RebuildManyAsync(string filter, Func<Guid, Task> action)
{
var handledIds = new HashSet<Guid>();
var worker = new ActionBlock<Guid>(action,
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 32
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
});
await eventStore.QueryAsync(async storedEvent =>
using (localCache.StartContext())
{
var headers = storedEvent.Data.Headers;
await store.GetSnapshotStore<TState>().ClearAsync();
var id = headers.AggregateId();
await eventStore.QueryAsync(async storedEvent =>
{
var id = storedEvent.Data.Headers.AggregateId();
if (handledIds.Add(id))
{
await worker.SendAsync(id);
}
}, filter, ct: CancellationToken.None);
}, filter);
worker.Complete();
await worker.Completion;
}
private async Task RebuildAsync<TState, TGrain>(Guid key, Func<Envelope<IEvent>, TState, TState> func) where TState : IDomainState, new()
{
var state = new TState
{
Version = EtagVersion.Empty
};
var persistence = store.WithSnapshotsAndEventSourcing(typeof(TGrain), key, (TState s) => state = s, e =>
{
state = func(e, state);
state.Version++;
});
await persistence.ReadAsync();
await persistence.WriteSnapshotAsync(state);
}
}
}

2
tools/Migrate_01/SquidexMigrations.cs

@ -7,6 +7,8 @@
using System.Reflection;
#pragma warning disable RECS0014 // If all fields, properties and methods members are static, the class can be made static.
namespace Migrate_01
{
public sealed class SquidexMigrations

Loading…
Cancel
Save