From 2db4ab91d3940c40f5253a44a304201f4b60fe6a Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Mon, 11 Dec 2017 09:02:05 +0100 Subject: [PATCH] Migrator. --- Squidex.sln | 2 +- .../AppProvider.cs | 1 - .../Schemas/SchemaCommandMiddleware.cs | 1 - .../Migrations/MongoMigrationEntity.cs | 29 +++++ .../Migrations/MongoMigrationStatus.cs | 66 +++++++++++ .../MongoDb/MongoRepositoryBase.cs | 2 +- .../Migrations/IMigration.cs | 21 ++++ .../Migrations/IMigrationStatus.cs | 21 ++++ .../Migrations/Migrator.cs | 100 ++++++++++++++++ src/Squidex/Config/Domain/ReadServices.cs | 1 - src/Squidex/Config/Domain/StoreServices.cs | 5 + src/Squidex/Config/Domain/SystemExtensions.cs | 8 ++ src/Squidex/WebStartup.cs | 1 + .../Schemas/Guards/GuardSchemaTests.cs | 2 - .../Migrations/MigratorTests.cs | 111 ++++++++++++++++++ .../Migrate_00.csproj} | 0 tools/{Migrate_01 => Migrate_00}/Program.cs | 2 +- 17 files changed, 365 insertions(+), 8 deletions(-) create mode 100644 src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationEntity.cs create mode 100644 src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationStatus.cs create mode 100644 src/Squidex.Infrastructure/Migrations/IMigration.cs create mode 100644 src/Squidex.Infrastructure/Migrations/IMigrationStatus.cs create mode 100644 src/Squidex.Infrastructure/Migrations/Migrator.cs create mode 100644 tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs rename tools/{Migrate_01/Migrate_01.csproj => Migrate_00/Migrate_00.csproj} (100%) rename tools/{Migrate_01 => Migrate_00}/Program.cs (99%) diff --git a/Squidex.sln b/Squidex.sln index 4d49d682e..cb23f22ff 100644 --- a/Squidex.sln +++ b/Squidex.sln @@ -36,7 +36,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.Goog EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "migrations", "migrations", "{94207AA6-4923-4183-A558-E0F8196B8CA3}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Migrate_01", "tools\Migrate_01\Migrate_01.csproj", "{B51126A8-0D75-4A79-867D-10724EC6AC84}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Migrate_00", "tools\Migrate_00\Migrate_00.csproj", "{B51126A8-0D75-4A79-867D-10724EC6AC84}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Shared", "src\Squidex.Shared\Squidex.Shared.csproj", "{5E75AB7D-6F01-4313-AFF1-7F7128FFD71F}" EndProject diff --git a/src/Squidex.Domain.Apps.Entities/AppProvider.cs b/src/Squidex.Domain.Apps.Entities/AppProvider.cs index 0f35e958f..55c030912 100644 --- a/src/Squidex.Domain.Apps.Entities/AppProvider.cs +++ b/src/Squidex.Domain.Apps.Entities/AppProvider.cs @@ -7,7 +7,6 @@ // ========================================================================== using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/SchemaCommandMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Schemas/SchemaCommandMiddleware.cs index cb5e13e38..bcb2c632d 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/SchemaCommandMiddleware.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/SchemaCommandMiddleware.cs @@ -9,7 +9,6 @@ using System; using System.Linq; using System.Threading.Tasks; -using Squidex.Domain.Apps.Entities.Schemas; using Squidex.Domain.Apps.Entities.Schemas.Commands; using Squidex.Domain.Apps.Entities.Schemas.Guards; using Squidex.Infrastructure; diff --git a/src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationEntity.cs b/src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationEntity.cs new file mode 100644 index 000000000..1a7161f68 --- /dev/null +++ b/src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationEntity.cs @@ -0,0 +1,29 @@ +// ========================================================================== +// MongoMigrationEntity.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace Squidex.Infrastructure.Migrations +{ + public sealed class MongoMigrationEntity + { + [BsonId] + [BsonElement] + [BsonRepresentation(BsonType.String)] + public string Id { get; set; } + + [BsonElement] + [BsonRequired] + public bool IsLocked { get; set; } + + [BsonElement] + [BsonRequired] + public int Version { get; set; } + } +} diff --git a/src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationStatus.cs b/src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationStatus.cs new file mode 100644 index 000000000..e81626185 --- /dev/null +++ b/src/Squidex.Infrastructure.MongoDb/Migrations/MongoMigrationStatus.cs @@ -0,0 +1,66 @@ +// ========================================================================== +// MongoMigrationStatus.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System.Threading.Tasks; +using MongoDB.Driver; +using Squidex.Infrastructure.MongoDb; + +namespace Squidex.Infrastructure.Migrations +{ + public sealed class MongoMigrationStatus : MongoRepositoryBase, IMigrationStatus + { + private const string DefaultId = "Default"; + + public MongoMigrationStatus(IMongoDatabase database) + : base(database) + { + } + + public override void Connect() + { + base.Connect(); + } + + public async Task GetVersionAsync() + { + var entity = await Collection.Find(x => x.Id == DefaultId).FirstOrDefaultAsync(); + + if (entity == null) + { + try + { + await Collection.InsertOneAsync(new MongoMigrationEntity { Id = DefaultId }); + } + catch (MongoWriteException ex) + { + if (ex.WriteError.Category != ServerErrorCategory.DuplicateKey) + { + throw; + } + } + } + + return entity?.Version ?? 0; + } + + public async Task TryLockAsync() + { + var entity = await Collection.FindOneAndUpdateAsync(x => x.Id == DefaultId, Update.Set(x => x.IsLocked, true)); + + return entity?.IsLocked == false; + } + + public Task UnlockAsync(int newVersion) + { + return Collection.UpdateOneAsync(x => x.Id == DefaultId, + Update + .Set(x => x.IsLocked, false) + .Set(x => x.Version, newVersion)); + } + } +} diff --git a/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoRepositoryBase.cs b/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoRepositoryBase.cs index 056dd8595..5183bc696 100644 --- a/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoRepositoryBase.cs +++ b/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoRepositoryBase.cs @@ -106,7 +106,7 @@ namespace Squidex.Infrastructure.MongoDb } } - public void Connect() + public virtual void Connect() { try { diff --git a/src/Squidex.Infrastructure/Migrations/IMigration.cs b/src/Squidex.Infrastructure/Migrations/IMigration.cs new file mode 100644 index 000000000..5d6fa7a6a --- /dev/null +++ b/src/Squidex.Infrastructure/Migrations/IMigration.cs @@ -0,0 +1,21 @@ +// ========================================================================== +// IMigration.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System.Threading.Tasks; + +namespace Squidex.Infrastructure.Migrations +{ + public interface IMigration + { + int FromVersion { get; } + + int ToVersion { get; } + + Task UpdateAsync(); + } +} diff --git a/src/Squidex.Infrastructure/Migrations/IMigrationStatus.cs b/src/Squidex.Infrastructure/Migrations/IMigrationStatus.cs new file mode 100644 index 000000000..96f7e2043 --- /dev/null +++ b/src/Squidex.Infrastructure/Migrations/IMigrationStatus.cs @@ -0,0 +1,21 @@ +// ========================================================================== +// IMigrationState.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System.Threading.Tasks; + +namespace Squidex.Infrastructure.Migrations +{ + public interface IMigrationStatus + { + Task GetVersionAsync(); + + Task TryLockAsync(); + + Task UnlockAsync(int newVersion); + } +} diff --git a/src/Squidex.Infrastructure/Migrations/Migrator.cs b/src/Squidex.Infrastructure/Migrations/Migrator.cs new file mode 100644 index 000000000..0be44904a --- /dev/null +++ b/src/Squidex.Infrastructure/Migrations/Migrator.cs @@ -0,0 +1,100 @@ +// ========================================================================== +// Migrator.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Squidex.Infrastructure.Log; + +namespace Squidex.Infrastructure.Migrations +{ + public sealed class Migrator + { + private readonly IMigrationStatus migrationStatus; + private readonly IEnumerable migrations; + private readonly ISemanticLog log; + + public Migrator(IMigrationStatus migrationStatus, IEnumerable migrations, ISemanticLog log) + { + Guard.NotNull(migrationStatus, nameof(migrationStatus)); + Guard.NotNull(migrations, nameof(migrations)); + Guard.NotNull(log, nameof(log)); + + this.migrationStatus = migrationStatus; + this.migrations = migrations.OrderByDescending(x => x.ToVersion).ToList(); + + this.log = log; + } + + public async Task MigrateAsync() + { + var version = await migrationStatus.GetVersionAsync(); + + var lastMigrator = migrations.FirstOrDefault(); + + if (lastMigrator != null && lastMigrator.ToVersion != version) + { + while (!await migrationStatus.TryLockAsync()) + { + log.LogInformation(w => w + .WriteProperty("action", "Migrate") + .WriteProperty("mesage", "Waiting 5sec to acquire lock.")); + + await Task.Delay(5000); + } + + try + { + var migrationPath = FindMigratorPath(version, lastMigrator.ToVersion).ToList(); + + foreach (var migrator in migrationPath) + { + using (log.MeasureInformation(w => w + .WriteProperty("action", "Migration") + .WriteProperty("migrator", migrator.GetType().ToString()))) + { + await migrator.UpdateAsync(); + + version = migrator.ToVersion; + } + } + } + finally + { + await migrationStatus.UnlockAsync(version); + } + } + } + + private IEnumerable FindMigratorPath(int fromVersion, int toVersion) + { + var addedMigrators = new HashSet(); + + while (true) + { + var bestMigrator = migrations.Where(x => x.FromVersion < x.ToVersion).FirstOrDefault(x => x.FromVersion == fromVersion); + + if (bestMigrator != null && addedMigrators.Add(bestMigrator)) + { + fromVersion = bestMigrator.ToVersion; + + yield return bestMigrator; + } + else if (fromVersion != toVersion) + { + throw new InvalidOperationException($"There is no migration path from {fromVersion} to {toVersion}."); + } + else + { + break; + } + } + } + } +} diff --git a/src/Squidex/Config/Domain/ReadServices.cs b/src/Squidex/Config/Domain/ReadServices.cs index d11421a78..2fd571e14 100644 --- a/src/Squidex/Config/Domain/ReadServices.cs +++ b/src/Squidex/Config/Domain/ReadServices.cs @@ -17,7 +17,6 @@ using Squidex.Domain.Apps.Entities; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Apps.Services; using Squidex.Domain.Apps.Entities.Apps.Services.Implementations; -using Squidex.Domain.Apps.Entities.Assets; using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Contents.Edm; using Squidex.Domain.Apps.Entities.Contents.GraphQL; diff --git a/src/Squidex/Config/Domain/StoreServices.cs b/src/Squidex/Config/Domain/StoreServices.cs index 6f743c789..416402e05 100644 --- a/src/Squidex/Config/Domain/StoreServices.cs +++ b/src/Squidex/Config/Domain/StoreServices.cs @@ -39,6 +39,7 @@ using Squidex.Domain.Users.MongoDb.Infrastructure; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.EventSourcing.Grains; +using Squidex.Infrastructure.Migrations; using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; using Squidex.Shared.Users; @@ -65,6 +66,10 @@ namespace Squidex.Config.Domain .As() .As(); + services.AddSingletonAs(c => new MongoMigrationStatus(mongoDatabase)) + .As() + .As(); + services.AddSingletonAs(c => new MongoSnapshotStore(mongoDatabase, c.GetRequiredService())) .As>() .As(); diff --git a/src/Squidex/Config/Domain/SystemExtensions.cs b/src/Squidex/Config/Domain/SystemExtensions.cs index ed8a4bccd..95538ef4c 100644 --- a/src/Squidex/Config/Domain/SystemExtensions.cs +++ b/src/Squidex/Config/Domain/SystemExtensions.cs @@ -10,6 +10,7 @@ using System; using System.Collections.Generic; using Microsoft.Extensions.DependencyInjection; using Squidex.Infrastructure; +using Squidex.Infrastructure.Migrations; namespace Squidex.Config.Domain { @@ -24,5 +25,12 @@ namespace Squidex.Config.Domain system.Connect(); } } + + public static void Migrate(this IServiceProvider services) + { + var migrator = services.GetRequiredService(); + + migrator.MigrateAsync().Wait(); + } } } diff --git a/src/Squidex/WebStartup.cs b/src/Squidex/WebStartup.cs index 2abdbadc3..fef509b0b 100644 --- a/src/Squidex/WebStartup.cs +++ b/src/Squidex/WebStartup.cs @@ -40,6 +40,7 @@ namespace Squidex { app.ApplicationServices.LogConfiguration(); app.ApplicationServices.TestExternalSystems(); + app.ApplicationServices.Migrate(); app.UseMyCors(); app.UseMyForwardingRules(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Guards/GuardSchemaTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Guards/GuardSchemaTests.cs index 326149595..be2bf2775 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Guards/GuardSchemaTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Guards/GuardSchemaTests.cs @@ -12,8 +12,6 @@ using System.Threading.Tasks; using FakeItEasy; using Squidex.Domain.Apps.Core; using Squidex.Domain.Apps.Core.Schemas; -using Squidex.Domain.Apps.Entities; -using Squidex.Domain.Apps.Entities.Schemas; using Squidex.Domain.Apps.Entities.Schemas.Commands; using Squidex.Infrastructure; using Xunit; diff --git a/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs b/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs new file mode 100644 index 000000000..adf71d3a5 --- /dev/null +++ b/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs @@ -0,0 +1,111 @@ +// ========================================================================== +// MigratorTests.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using FakeItEasy; +using Squidex.Infrastructure.Log; +using Xunit; + +namespace Squidex.Infrastructure.Migrations +{ + public sealed class MigratorTests + { + private readonly IMigrationStatus status = A.Fake(); + + public MigratorTests() + { + A.CallTo(() => status.GetVersionAsync()).Returns(0); + A.CallTo(() => status.TryLockAsync()).Returns(true); + } + + [Fact] + public async Task Should_migrate_step_by_step() + { + var migrator_0_1 = BuildMigration(0, 1); + var migrator_1_2 = BuildMigration(1, 2); + var migrator_2_3 = BuildMigration(2, 3); + + var migrator = new Migrator(status, new[] { migrator_0_1, migrator_1_2, migrator_2_3 }, A.Fake()); + + await migrator.MigrateAsync(); + + A.CallTo(() => migrator_0_1.UpdateAsync()).MustHaveHappened(); + A.CallTo(() => migrator_1_2.UpdateAsync()).MustHaveHappened(); + A.CallTo(() => migrator_2_3.UpdateAsync()).MustHaveHappened(); + + A.CallTo(() => status.UnlockAsync(3)).MustHaveHappened(); + } + + [Fact] + public async Task Should_unlock_when_failed() + { + var migrator_0_1 = BuildMigration(0, 1); + var migrator_1_2 = BuildMigration(1, 2); + var migrator_2_3 = BuildMigration(2, 3); + + var migrator = new Migrator(status, new[] { migrator_0_1, migrator_1_2, migrator_2_3 }, A.Fake()); + + A.CallTo(() => migrator_1_2.UpdateAsync()).Throws(new ArgumentException()); + + await Assert.ThrowsAsync(migrator.MigrateAsync); + + A.CallTo(() => migrator_0_1.UpdateAsync()).MustHaveHappened(); + A.CallTo(() => migrator_1_2.UpdateAsync()).MustHaveHappened(); + A.CallTo(() => migrator_2_3.UpdateAsync()).MustNotHaveHappened(); + + A.CallTo(() => status.UnlockAsync(1)).MustHaveHappened(); + } + + [Fact] + public async Task Should_migrate_with_fastest_path() + { + var migrator_0_1 = BuildMigration(0, 1); + var migrator_0_2 = BuildMigration(0, 2); + var migrator_1_2 = BuildMigration(1, 2); + var migrator_2_3 = BuildMigration(2, 3); + + var migrator = new Migrator(status, new[] { migrator_0_1, migrator_0_2, migrator_1_2, migrator_2_3 }, A.Fake()); + + await migrator.MigrateAsync(); + + A.CallTo(() => migrator_0_2.UpdateAsync()).MustHaveHappened(); + A.CallTo(() => migrator_0_1.UpdateAsync()).MustNotHaveHappened(); + A.CallTo(() => migrator_1_2.UpdateAsync()).MustNotHaveHappened(); + A.CallTo(() => migrator_2_3.UpdateAsync()).MustHaveHappened(); + + A.CallTo(() => status.UnlockAsync(3)).MustHaveHappened(); + } + + [Fact] + public async Task Should_throw_if_no_path_found() + { + var migrator_0_1 = BuildMigration(0, 1); + var migrator_2_3 = BuildMigration(2, 3); + + var migrator = new Migrator(status, new[] { migrator_0_1, migrator_2_3 }, A.Fake()); + + await Assert.ThrowsAsync(migrator.MigrateAsync); + + A.CallTo(() => migrator_0_1.UpdateAsync()).MustNotHaveHappened(); + A.CallTo(() => migrator_2_3.UpdateAsync()).MustNotHaveHappened(); + + A.CallTo(() => status.UnlockAsync(0)).MustHaveHappened(); + } + + private IMigration BuildMigration(int fromVersion, int toVersion) + { + var migration = A.Fake(); + + A.CallTo(() => migration.FromVersion).Returns(fromVersion); + A.CallTo(() => migration.ToVersion).Returns(toVersion); + + return migration; + } + } +} diff --git a/tools/Migrate_01/Migrate_01.csproj b/tools/Migrate_00/Migrate_00.csproj similarity index 100% rename from tools/Migrate_01/Migrate_01.csproj rename to tools/Migrate_00/Migrate_00.csproj diff --git a/tools/Migrate_01/Program.cs b/tools/Migrate_00/Program.cs similarity index 99% rename from tools/Migrate_01/Program.cs rename to tools/Migrate_00/Program.cs index 522763767..1429070b3 100644 --- a/tools/Migrate_01/Program.cs +++ b/tools/Migrate_00/Program.cs @@ -10,7 +10,7 @@ using System; using MongoDB.Bson; using MongoDB.Driver; -namespace Migrate_01 +namespace Migrate_00 { public class Program {