Browse Source

CancellationToken fix.

pull/711/head
Sebastian Stehle 5 years ago
parent
commit
4edba74c40
  1. 3
      backend/src/Migrations/Migrations/ClearRules.cs
  2. 3
      backend/src/Migrations/Migrations/ClearSchemas.cs
  3. 7
      backend/src/Migrations/Migrations/ConvertEventStore.cs
  4. 7
      backend/src/Migrations/Migrations/ConvertEventStoreAppId.cs
  5. 5
      backend/src/Migrations/Migrations/CreateAssetSlugs.cs
  6. 5
      backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs
  7. 17
      backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs
  8. 3
      backend/src/Migrations/Migrations/MongoDb/ConvertOldSnapshotStores.cs
  9. 7
      backend/src/Migrations/Migrations/MongoDb/ConvertRuleEventsJson.cs
  10. 11
      backend/src/Migrations/Migrations/MongoDb/DeleteContentCollections.cs
  11. 15
      backend/src/Migrations/Migrations/MongoDb/RenameAssetMetadata.cs
  12. 5
      backend/src/Migrations/Migrations/MongoDb/RenameAssetSlugField.cs
  13. 11
      backend/src/Migrations/Migrations/MongoDb/RestructureContentCollection.cs
  14. 21
      backend/src/Migrations/Migrations/PopulateGrainIndexes.cs
  15. 5
      backend/src/Migrations/Migrations/RebuildApps.cs
  16. 5
      backend/src/Migrations/Migrations/RebuildAssetFolders.cs
  17. 5
      backend/src/Migrations/Migrations/RebuildAssets.cs
  18. 5
      backend/src/Migrations/Migrations/RebuildContents.cs
  19. 15
      backend/src/Migrations/Migrations/RebuildSnapshots.cs
  20. 3
      backend/src/Migrations/Migrations/StartEventConsumers.cs
  21. 3
      backend/src/Migrations/Migrations/StopEventConsumers.cs
  22. 14
      backend/src/Migrations/RebuildRunner.cs
  23. 75
      backend/src/Squidex.Domain.Apps.Entities/Assets/RepairFiles.cs
  24. 3
      backend/src/Squidex.Infrastructure/Migrations/IMigration.cs
  25. 2
      backend/src/Squidex.Infrastructure/Migrations/Migrator.cs
  26. 2
      backend/src/Squidex/Config/Domain/AssetServices.cs
  27. 4
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RepairFilesTests.cs
  28. 29
      backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs

3
backend/src/Migrations/Migrations/ClearRules.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Rules.DomainObject; using Squidex.Domain.Apps.Entities.Rules.DomainObject;
using Squidex.Infrastructure.Migrations; using Squidex.Infrastructure.Migrations;
@ -21,7 +22,7 @@ namespace Migrations.Migrations
this.store = store; this.store = store;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return store.ClearSnapshotsAsync(); return store.ClearSnapshotsAsync();
} }

3
backend/src/Migrations/Migrations/ClearSchemas.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Schemas.DomainObject; using Squidex.Domain.Apps.Entities.Schemas.DomainObject;
using Squidex.Infrastructure.Migrations; using Squidex.Infrastructure.Migrations;
@ -21,7 +22,7 @@ namespace Migrations.Migrations
this.store = store; this.store = store;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return store.ClearSnapshotsAsync(); return store.ClearSnapshotsAsync();
} }

7
backend/src/Migrations/Migrations/ConvertEventStore.cs

@ -6,6 +6,7 @@
// ========================================================================== // ==========================================================================
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -23,7 +24,7 @@ namespace Migrations.Migrations
this.eventStore = eventStore; this.eventStore = eventStore;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
if (eventStore is MongoEventStore mongoEventStore) if (eventStore is MongoEventStore mongoEventStore)
{ {
@ -42,7 +43,7 @@ namespace Migrations.Migrations
if (writes.Count == 1000 || (force && writes.Count > 0)) if (writes.Count == 1000 || (force && writes.Count > 0))
{ {
await collection.BulkWriteAsync(writes); await collection.BulkWriteAsync(writes, cancellationToken: ct);
writes.Clear(); writes.Clear();
} }
@ -59,7 +60,7 @@ namespace Migrations.Migrations
} }
await WriteAsync(new ReplaceOneModel<BsonDocument>(filter.Eq("_id", commit["_id"].AsString), commit), false); await WriteAsync(new ReplaceOneModel<BsonDocument>(filter.Eq("_id", commit["_id"].AsString), commit), false);
}); }, ct);
await WriteAsync(null, true); await WriteAsync(null, true);
} }

7
backend/src/Migrations/Migrations/ConvertEventStoreAppId.cs

@ -7,6 +7,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -25,7 +26,7 @@ namespace Migrations.Migrations
this.eventStore = eventStore; this.eventStore = eventStore;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
if (eventStore is MongoEventStore mongoEventStore) if (eventStore is MongoEventStore mongoEventStore)
{ {
@ -45,7 +46,7 @@ namespace Migrations.Migrations
if (writesBatches.Count == 1000 || (force && writesBatches.Count > 0)) if (writesBatches.Count == 1000 || (force && writesBatches.Count > 0))
{ {
await collection.BulkWriteAsync(writesBatches); await collection.BulkWriteAsync(writesBatches, cancellationToken: ct);
writesBatches.Clear(); writesBatches.Clear();
} }
@ -86,7 +87,7 @@ namespace Migrations.Migrations
await WriteAsync(write, false); await WriteAsync(write, false);
} }
}); }, ct);
await WriteAsync(null, true); await WriteAsync(null, true);
} }

5
backend/src/Migrations/Migrations/CreateAssetSlugs.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Assets; using Squidex.Domain.Apps.Entities.Assets;
using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Domain.Apps.Entities.Assets.DomainObject;
@ -23,7 +24,7 @@ namespace Migrations.Migrations
this.stateForAssets = stateForAssets; this.stateForAssets = stateForAssets;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return stateForAssets.ReadAllAsync(async (state, version) => return stateForAssets.ReadAllAsync(async (state, version) =>
{ {
@ -32,7 +33,7 @@ namespace Migrations.Migrations
var key = DomainId.Combine(state.AppId.Id, state.Id); var key = DomainId.Combine(state.AppId.Id, state.Id);
await stateForAssets.WriteAsync(key, state, version, version); await stateForAssets.WriteAsync(key, state, version, version);
}); }, ct);
} }
} }
} }

5
backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs

@ -8,6 +8,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis; using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow; using System.Threading.Tasks.Dataflow;
using MongoDB.Bson; using MongoDB.Bson;
@ -26,7 +27,7 @@ namespace Migrations.Migrations.MongoDb
this.database = database; this.database = database;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
const int SizeOfBatch = 1000; const int SizeOfBatch = 1000;
const int SizeOfQueue = 20; const int SizeOfQueue = 20;
@ -122,7 +123,7 @@ namespace Migrations.Migrations.MongoDb
PropagateCompletion = true PropagateCompletion = true
}); });
await collectionOld.Find(new BsonDocument()).ForEachAsync(batchBlock.SendAsync); await collectionOld.Find(new BsonDocument()).ForEachAsync(batchBlock.SendAsync, ct);
batchBlock.Complete(); batchBlock.Complete();

17
backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs

@ -7,6 +7,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow; using System.Threading.Tasks.Dataflow;
using MongoDB.Bson; using MongoDB.Bson;
@ -55,22 +56,22 @@ namespace Migrations.Migrations.MongoDb
return this; return this;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
switch (scope) switch (scope)
{ {
case Scope.Assets: case Scope.Assets:
await RebuildAsync(database, ConvertParentId, "States_Assets"); await RebuildAsync(database, ConvertParentId, "States_Assets", ct);
await RebuildAsync(database, ConvertParentId, "States_AssetFolders"); await RebuildAsync(database, ConvertParentId, "States_AssetFolders", ct);
break; break;
case Scope.Contents: case Scope.Contents:
await RebuildAsync(databaseContent, null, "State_Contents_All"); await RebuildAsync(databaseContent, null, "State_Contents_All", ct);
await RebuildAsync(databaseContent, null, "State_Contents_Published"); await RebuildAsync(databaseContent, null, "State_Contents_Published", ct);
break; break;
} }
} }
private static async Task RebuildAsync(IMongoDatabase database, Action<BsonDocument>? extraAction, string collectionNameOld) private static async Task RebuildAsync(IMongoDatabase database, Action<BsonDocument>? extraAction, string collectionNameOld, CancellationToken ct)
{ {
const int SizeOfBatch = 1000; const int SizeOfBatch = 1000;
const int SizeOfQueue = 10; const int SizeOfQueue = 10;
@ -88,7 +89,7 @@ namespace Migrations.Migrations.MongoDb
return; return;
} }
await collectionNew.DeleteManyAsync(new BsonDocument()); await collectionNew.DeleteManyAsync(new BsonDocument(), ct);
var batchBlock = new BatchBlock<BsonDocument>(SizeOfBatch, new GroupingDataflowBlockOptions var batchBlock = new BatchBlock<BsonDocument>(SizeOfBatch, new GroupingDataflowBlockOptions
{ {
@ -156,7 +157,7 @@ namespace Migrations.Migrations.MongoDb
PropagateCompletion = true PropagateCompletion = true
}); });
await collectionOld.Find(new BsonDocument()).ForEachAsync(batchBlock.SendAsync); await collectionOld.Find(new BsonDocument()).ForEachAsync(batchBlock.SendAsync, ct);
batchBlock.Complete(); batchBlock.Complete();

3
backend/src/Migrations/Migrations/MongoDb/ConvertOldSnapshotStores.cs

@ -6,6 +6,7 @@
// ========================================================================== // ==========================================================================
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -22,7 +23,7 @@ namespace Migrations.Migrations.MongoDb
this.database = database; this.database = database;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
var collections = new[] var collections = new[]
{ {

7
backend/src/Migrations/Migrations/MongoDb/ConvertRuleEventsJson.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -21,9 +22,9 @@ namespace Migrations.Migrations.MongoDb
collection = database.GetCollection<BsonDocument>("RuleEvents"); collection = database.GetCollection<BsonDocument>("RuleEvents");
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
foreach (var document in collection.Find(new BsonDocument()).ToEnumerable()) foreach (var document in collection.Find(new BsonDocument()).ToEnumerable(ct))
{ {
try try
{ {
@ -31,7 +32,7 @@ namespace Migrations.Migrations.MongoDb
var filter = Builders<BsonDocument>.Filter.Eq("_id", document["_id"].ToString()); var filter = Builders<BsonDocument>.Filter.Eq("_id", document["_id"].ToString());
await collection.ReplaceOneAsync(filter, document); await collection.ReplaceOneAsync(filter, document, cancellationToken: ct);
} }
catch catch
{ {

11
backend/src/Migrations/Migrations/MongoDb/DeleteContentCollections.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Driver; using MongoDB.Driver;
using Squidex.Infrastructure.Migrations; using Squidex.Infrastructure.Migrations;
@ -20,12 +21,12 @@ namespace Migrations.Migrations.MongoDb
this.database = database; this.database = database;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
await database.DropCollectionAsync("States_Contents"); await database.DropCollectionAsync("States_Contents", ct);
await database.DropCollectionAsync("States_Contents_Archive"); await database.DropCollectionAsync("States_Contents_Archive", ct);
await database.DropCollectionAsync("State_Content_Draft"); await database.DropCollectionAsync("State_Content_Draft", ct);
await database.DropCollectionAsync("State_Content_Published"); await database.DropCollectionAsync("State_Content_Published", ct);
} }
} }
} }

15
backend/src/Migrations/Migrations/MongoDb/RenameAssetMetadata.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -21,7 +22,7 @@ namespace Migrations.Migrations.MongoDb
this.database = database; this.database = database;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
var collection = database.GetCollection<BsonDocument>("States_Assets"); var collection = database.GetCollection<BsonDocument>("States_Assets");
@ -29,39 +30,39 @@ namespace Migrations.Migrations.MongoDb
Builders<BsonDocument>.Update Builders<BsonDocument>.Update
.Set("md", new BsonDocument()); .Set("md", new BsonDocument());
await collection.UpdateManyAsync(new BsonDocument(), createMetadata); await collection.UpdateManyAsync(new BsonDocument(), createMetadata, cancellationToken: ct);
var removeNullPixelInfos = var removeNullPixelInfos =
Builders<BsonDocument>.Update Builders<BsonDocument>.Update
.Unset("ph") .Unset("ph")
.Unset("pw"); .Unset("pw");
await collection.UpdateManyAsync(new BsonDocument("ph", BsonValue.Create(null)), removeNullPixelInfos); await collection.UpdateManyAsync(new BsonDocument("ph", BsonValue.Create(null)), removeNullPixelInfos, cancellationToken: ct);
var setPixelDimensions = var setPixelDimensions =
Builders<BsonDocument>.Update Builders<BsonDocument>.Update
.Rename("ph", "md.pixelHeight") .Rename("ph", "md.pixelHeight")
.Rename("pw", "md.pixelWidth"); .Rename("pw", "md.pixelWidth");
await collection.UpdateManyAsync(new BsonDocument(), setPixelDimensions); await collection.UpdateManyAsync(new BsonDocument(), setPixelDimensions, cancellationToken: ct);
var setTypeToImage = var setTypeToImage =
Builders<BsonDocument>.Update Builders<BsonDocument>.Update
.Set("at", "Image"); .Set("at", "Image");
await collection.UpdateManyAsync(new BsonDocument("im", true), setTypeToImage); await collection.UpdateManyAsync(new BsonDocument("im", true), setTypeToImage, cancellationToken: ct);
var setTypeToUnknown = var setTypeToUnknown =
Builders<BsonDocument>.Update Builders<BsonDocument>.Update
.Set("at", "Unknown"); .Set("at", "Unknown");
await collection.UpdateManyAsync(new BsonDocument("im", false), setTypeToUnknown); await collection.UpdateManyAsync(new BsonDocument("im", false), setTypeToUnknown, cancellationToken: ct);
var removeIsImage = var removeIsImage =
Builders<BsonDocument>.Update Builders<BsonDocument>.Update
.Unset("im"); .Unset("im");
await collection.UpdateManyAsync(new BsonDocument(), removeIsImage); await collection.UpdateManyAsync(new BsonDocument(), removeIsImage, cancellationToken: ct);
} }
} }
} }

5
backend/src/Migrations/Migrations/MongoDb/RenameAssetSlugField.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -21,13 +22,13 @@ namespace Migrations.Migrations.MongoDb
this.database = database; this.database = database;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
var collection = database.GetCollection<BsonDocument>("States_Assets"); var collection = database.GetCollection<BsonDocument>("States_Assets");
var update = Builders<BsonDocument>.Update.Rename("FileNameSlug", "Slug"); var update = Builders<BsonDocument>.Update.Rename("FileNameSlug", "Slug");
return collection.UpdateManyAsync(new BsonDocument(), update); return collection.UpdateManyAsync(new BsonDocument(), update, cancellationToken: ct);
} }
} }
} }

11
backend/src/Migrations/Migrations/MongoDb/RestructureContentCollection.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Driver; using MongoDB.Driver;
@ -22,20 +23,20 @@ namespace Migrations.Migrations.MongoDb
this.contentDatabase = contentDatabase; this.contentDatabase = contentDatabase;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
if (await contentDatabase.CollectionExistsAsync("State_Content_Draft")) if (await contentDatabase.CollectionExistsAsync("State_Content_Draft"))
{ {
await contentDatabase.DropCollectionAsync("State_Contents"); await contentDatabase.DropCollectionAsync("State_Contents", ct);
await contentDatabase.DropCollectionAsync("State_Content_Published"); await contentDatabase.DropCollectionAsync("State_Content_Published", ct);
await contentDatabase.RenameCollectionAsync("State_Content_Draft", "State_Contents"); await contentDatabase.RenameCollectionAsync("State_Content_Draft", "State_Contents", cancellationToken: ct);
} }
if (await contentDatabase.CollectionExistsAsync("State_Contents")) if (await contentDatabase.CollectionExistsAsync("State_Contents"))
{ {
var collection = contentDatabase.GetCollection<BsonDocument>("State_Contents"); var collection = contentDatabase.GetCollection<BsonDocument>("State_Contents");
await collection.UpdateManyAsync(new BsonDocument(), Builders<BsonDocument>.Update.Unset("dt")); await collection.UpdateManyAsync(new BsonDocument(), Builders<BsonDocument>.Update.Unset("dt"), cancellationToken: ct);
} }
} }
} }

21
backend/src/Migrations/Migrations/PopulateGrainIndexes.cs

@ -6,6 +6,7 @@
// ========================================================================== // ==========================================================================
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Apps.Indexes; using Squidex.Domain.Apps.Entities.Apps.Indexes;
using Squidex.Domain.Apps.Entities.Rules.Indexes; using Squidex.Domain.Apps.Entities.Rules.Indexes;
@ -39,15 +40,15 @@ namespace Migrations.Migrations
this.eventStore = eventStore; this.eventStore = eventStore;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return Task.WhenAll( return Task.WhenAll(
RebuildAppIndexes(), RebuildAppIndexes(ct),
RebuildRuleIndexes(), RebuildRuleIndexes(ct),
RebuildSchemaIndexes()); RebuildSchemaIndexes(ct));
} }
private async Task RebuildAppIndexes() private async Task RebuildAppIndexes(CancellationToken ct)
{ {
var appsByName = new Dictionary<string, DomainId>(); var appsByName = new Dictionary<string, DomainId>();
var appsByUser = new Dictionary<string, HashSet<DomainId>>(); var appsByUser = new Dictionary<string, HashSet<DomainId>>();
@ -75,7 +76,7 @@ namespace Migrations.Migrations
} }
} }
await foreach (var storedEvent in eventStore.QueryAllAsync("^app\\-")) await foreach (var storedEvent in eventStore.QueryAllAsync("^app\\-", ct: ct))
{ {
var @event = eventDataFormatter.ParseIfKnown(storedEvent); var @event = eventDataFormatter.ParseIfKnown(storedEvent);
@ -119,7 +120,7 @@ namespace Migrations.Migrations
} }
} }
private async Task RebuildRuleIndexes() private async Task RebuildRuleIndexes(CancellationToken ct)
{ {
var rulesByApp = new Dictionary<DomainId, HashSet<DomainId>>(); var rulesByApp = new Dictionary<DomainId, HashSet<DomainId>>();
@ -128,7 +129,7 @@ namespace Migrations.Migrations
return rulesByApp!.GetOrAddNew(@event.AppId.Id); return rulesByApp!.GetOrAddNew(@event.AppId.Id);
} }
await foreach (var storedEvent in eventStore.QueryAllAsync("^rule\\-")) await foreach (var storedEvent in eventStore.QueryAllAsync("^rule\\-", ct: ct))
{ {
var @event = eventDataFormatter.ParseIfKnown(storedEvent); var @event = eventDataFormatter.ParseIfKnown(storedEvent);
@ -152,7 +153,7 @@ namespace Migrations.Migrations
} }
} }
private async Task RebuildSchemaIndexes() private async Task RebuildSchemaIndexes(CancellationToken ct)
{ {
var schemasByApp = new Dictionary<DomainId, Dictionary<string, DomainId>>(); var schemasByApp = new Dictionary<DomainId, Dictionary<string, DomainId>>();
@ -161,7 +162,7 @@ namespace Migrations.Migrations
return schemasByApp!.GetOrAddNew(@event.AppId.Id); return schemasByApp!.GetOrAddNew(@event.AppId.Id);
} }
await foreach (var storedEvent in eventStore.QueryAllAsync("^schema\\-")) await foreach (var storedEvent in eventStore.QueryAllAsync("^schema\\-", ct: ct))
{ {
var @event = eventDataFormatter.ParseIfKnown(storedEvent); var @event = eventDataFormatter.ParseIfKnown(storedEvent);

5
backend/src/Migrations/Migrations/RebuildApps.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Commands;
@ -24,9 +25,9 @@ namespace Migrations.Migrations
this.rebuildOptions = rebuildOptions.Value; this.rebuildOptions = rebuildOptions.Value;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return rebuilder.RebuildAppsAsync(rebuildOptions.BatchSize); return rebuilder.RebuildAppsAsync(rebuildOptions.BatchSize, ct);
} }
} }
} }

5
backend/src/Migrations/Migrations/RebuildAssetFolders.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Commands;
@ -24,9 +25,9 @@ namespace Migrations.Migrations
this.rebuildOptions = rebuildOptions.Value; this.rebuildOptions = rebuildOptions.Value;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return rebuilder.RebuildAssetFoldersAsync(rebuildOptions.BatchSize); return rebuilder.RebuildAssetFoldersAsync(rebuildOptions.BatchSize, ct);
} }
} }
} }

5
backend/src/Migrations/Migrations/RebuildAssets.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Commands;
@ -24,9 +25,9 @@ namespace Migrations.Migrations
this.rebuildOptions = rebuildOptions.Value; this.rebuildOptions = rebuildOptions.Value;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return rebuilder.RebuildAssetsAsync(rebuildOptions.BatchSize); return rebuilder.RebuildAssetsAsync(rebuildOptions.BatchSize, ct);
} }
} }
} }

5
backend/src/Migrations/Migrations/RebuildContents.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Commands;
@ -24,9 +25,9 @@ namespace Migrations.Migrations
this.rebuildOptions = rebuildOptions.Value; this.rebuildOptions = rebuildOptions.Value;
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return rebuilder.RebuildContentAsync(rebuildOptions.BatchSize); return rebuilder.RebuildContentAsync(rebuildOptions.BatchSize, ct);
} }
} }
} }

15
backend/src/Migrations/Migrations/RebuildSnapshots.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Commands;
@ -24,14 +25,14 @@ namespace Migrations.Migrations
this.rebuildOptions = rebuildOptions.Value; this.rebuildOptions = rebuildOptions.Value;
} }
public async Task UpdateAsync() public async Task UpdateAsync(CancellationToken ct)
{ {
await rebuilder.RebuildAppsAsync(rebuildOptions.BatchSize); await rebuilder.RebuildAppsAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildSchemasAsync(rebuildOptions.BatchSize); await rebuilder.RebuildSchemasAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildRulesAsync(rebuildOptions.BatchSize); await rebuilder.RebuildRulesAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildContentAsync(rebuildOptions.BatchSize); await rebuilder.RebuildContentAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildAssetsAsync(rebuildOptions.BatchSize); await rebuilder.RebuildAssetsAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildAssetFoldersAsync(rebuildOptions.BatchSize); await rebuilder.RebuildAssetFoldersAsync(rebuildOptions.BatchSize, ct);
} }
} }
} }

3
backend/src/Migrations/Migrations/StartEventConsumers.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Orleans; using Orleans;
using Squidex.Infrastructure.EventSourcing.Grains; using Squidex.Infrastructure.EventSourcing.Grains;
@ -22,7 +23,7 @@ namespace Migrations.Migrations
eventConsumerManager = grainFactory.GetGrain<IEventConsumerManagerGrain>(SingleGrain.Id); eventConsumerManager = grainFactory.GetGrain<IEventConsumerManagerGrain>(SingleGrain.Id);
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return eventConsumerManager.StartAllAsync(); return eventConsumerManager.StartAllAsync();
} }

3
backend/src/Migrations/Migrations/StopEventConsumers.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Orleans; using Orleans;
using Squidex.Infrastructure.EventSourcing.Grains; using Squidex.Infrastructure.EventSourcing.Grains;
@ -22,7 +23,7 @@ namespace Migrations.Migrations
eventConsumerManager = grainFactory.GetGrain<IEventConsumerManagerGrain>(SingleGrain.Id); eventConsumerManager = grainFactory.GetGrain<IEventConsumerManagerGrain>(SingleGrain.Id);
} }
public Task UpdateAsync() public Task UpdateAsync(CancellationToken ct)
{ {
return eventConsumerManager.StopAllAsync(); return eventConsumerManager.StopAllAsync();
} }

14
backend/src/Migrations/RebuildRunner.cs

@ -17,23 +17,23 @@ namespace Migrations
{ {
public sealed class RebuildRunner public sealed class RebuildRunner
{ {
private readonly RepairFiles repairFiles; private readonly RebuildFiles rebuildFiles;
private readonly Rebuilder rebuilder; private readonly Rebuilder rebuilder;
private readonly PopulateGrainIndexes populateGrainIndexes; private readonly PopulateGrainIndexes populateGrainIndexes;
private readonly RebuildOptions rebuildOptions; private readonly RebuildOptions rebuildOptions;
public RebuildRunner( public RebuildRunner(
RepairFiles repairFiles,
Rebuilder rebuilder,
IOptions<RebuildOptions> rebuildOptions, IOptions<RebuildOptions> rebuildOptions,
Rebuilder rebuilder,
RebuildFiles rebuildFiles,
PopulateGrainIndexes populateGrainIndexes) PopulateGrainIndexes populateGrainIndexes)
{ {
Guard.NotNull(repairFiles, nameof(repairFiles)); Guard.NotNull(rebuildFiles, nameof(rebuildFiles));
Guard.NotNull(rebuilder, nameof(rebuilder)); Guard.NotNull(rebuilder, nameof(rebuilder));
Guard.NotNull(rebuildOptions, nameof(rebuildOptions)); Guard.NotNull(rebuildOptions, nameof(rebuildOptions));
Guard.NotNull(populateGrainIndexes, nameof(populateGrainIndexes)); Guard.NotNull(populateGrainIndexes, nameof(populateGrainIndexes));
this.repairFiles = repairFiles; this.rebuildFiles = rebuildFiles;
this.rebuilder = rebuilder; this.rebuilder = rebuilder;
this.rebuildOptions = rebuildOptions.Value; this.rebuildOptions = rebuildOptions.Value;
this.populateGrainIndexes = populateGrainIndexes; this.populateGrainIndexes = populateGrainIndexes;
@ -66,7 +66,7 @@ namespace Migrations
if (rebuildOptions.AssetFiles) if (rebuildOptions.AssetFiles)
{ {
await repairFiles.RepairAsync(ct); await rebuildFiles.RepairAsync(ct);
} }
if (rebuildOptions.Contents) if (rebuildOptions.Contents)
@ -76,7 +76,7 @@ namespace Migrations
if (rebuildOptions.Indexes) if (rebuildOptions.Indexes)
{ {
await populateGrainIndexes.UpdateAsync(); await populateGrainIndexes.UpdateAsync(ct);
} }
} }
} }

75
backend/src/Squidex.Domain.Apps.Entities/Assets/RepairFiles.cs

@ -1,75 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Squidex.Assets;
using Squidex.Domain.Apps.Events.Assets;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Domain.Apps.Entities.Assets
{
public sealed class RepairFiles
{
private static readonly MemoryStream DummyStream = new MemoryStream(Encoding.UTF8.GetBytes("dummy"));
private readonly IAssetFileStore assetFileStore;
private readonly IEventStore eventStore;
private readonly IEventDataFormatter eventDataFormatter;
public RepairFiles(
IAssetFileStore assetFileStore,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter)
{
Guard.NotNull(assetFileStore, nameof(assetFileStore));
Guard.NotNull(eventStore, nameof(eventStore));
Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter));
this.assetFileStore = assetFileStore;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
}
public async Task RepairAsync(CancellationToken ct = default)
{
await foreach (var storedEvent in eventStore.QueryAllAsync("^asset\\-", ct: ct))
{
var @event = eventDataFormatter.ParseIfKnown(storedEvent);
if (@event != null)
{
switch (@event.Payload)
{
case AssetCreated assetCreated:
await TryRepairAsync(assetCreated.AppId, assetCreated.AssetId, assetCreated.FileVersion, ct);
break;
case AssetUpdated assetUpdated:
await TryRepairAsync(assetUpdated.AppId, assetUpdated.AssetId, assetUpdated.FileVersion, ct);
break;
}
}
}
}
private async Task TryRepairAsync(NamedId<DomainId> appId, DomainId id, long fileVersion, CancellationToken ct)
{
try
{
await assetFileStore.GetFileSizeAsync(appId.Id, id, fileVersion, ct);
}
catch (AssetNotFoundException)
{
DummyStream.Position = 0;
await assetFileStore.UploadAsync(appId.Id, id, fileVersion, DummyStream, ct);
}
}
}
}

3
backend/src/Squidex.Infrastructure/Migrations/IMigration.cs

@ -5,12 +5,13 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace Squidex.Infrastructure.Migrations namespace Squidex.Infrastructure.Migrations
{ {
public interface IMigration public interface IMigration
{ {
Task UpdateAsync(); Task UpdateAsync(CancellationToken ct);
} }
} }

2
backend/src/Squidex.Infrastructure/Migrations/Migrator.cs

@ -73,7 +73,7 @@ namespace Squidex.Infrastructure.Migrations
.WriteProperty("status", "Completed") .WriteProperty("status", "Completed")
.WriteProperty("migrator", name))) .WriteProperty("migrator", name)))
{ {
await migration.UpdateAsync(); await migration.UpdateAsync(ct);
} }
} }
catch (Exception ex) catch (Exception ex)

2
backend/src/Squidex/Config/Domain/AssetServices.cs

@ -53,7 +53,7 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<AssetQueryParser>() services.AddSingletonAs<AssetQueryParser>()
.AsSelf(); .AsSelf();
services.AddSingletonAs<RepairFiles>() services.AddSingletonAs<RebuildFiles>()
.AsSelf(); .AsSelf();
services.AddTransientAs<AssetHistoryEventsCreator>() services.AddTransientAs<AssetHistoryEventsCreator>()

4
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RepairFilesTests.cs

@ -24,11 +24,11 @@ namespace Squidex.Domain.Apps.Entities.Assets
private readonly IEventDataFormatter eventDataFormatter = A.Fake<IEventDataFormatter>(); private readonly IEventDataFormatter eventDataFormatter = A.Fake<IEventDataFormatter>();
private readonly IAssetFileStore assetFileStore = A.Fake<IAssetFileStore>(); private readonly IAssetFileStore assetFileStore = A.Fake<IAssetFileStore>();
private readonly NamedId<DomainId> appId = NamedId.Of(DomainId.NewGuid(), "my-app"); private readonly NamedId<DomainId> appId = NamedId.Of(DomainId.NewGuid(), "my-app");
private readonly RepairFiles sut; private readonly RebuildFiles sut;
public RepairFilesTests() public RepairFilesTests()
{ {
sut = new RepairFiles(assetFileStore, eventStore, eventDataFormatter); sut = new RebuildFiles(assetFileStore, eventStore, eventDataFormatter);
} }
[Fact] [Fact]

29
backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs

@ -8,6 +8,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using FakeItEasy; using FakeItEasy;
using Squidex.Log; using Squidex.Log;
@ -103,13 +104,13 @@ namespace Squidex.Infrastructure.Migrations
await sut.MigrateAsync(); await sut.MigrateAsync();
A.CallTo(() => migrator_0_1.UpdateAsync()) A.CallTo(() => migrator_0_1.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_1_2.UpdateAsync()) A.CallTo(() => migrator_1_2.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_2_3.UpdateAsync()) A.CallTo(() => migrator_2_3.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => status.CompleteAsync(1)) A.CallTo(() => status.CompleteAsync(1))
@ -136,13 +137,13 @@ namespace Squidex.Infrastructure.Migrations
await sut.MigrateAsync(); await sut.MigrateAsync();
A.CallTo(() => migrator_0_1.UpdateAsync()) A.CallTo(() => migrator_0_1.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_1_2.UpdateAsync()) A.CallTo(() => migrator_1_2.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_2_3.UpdateAsync()) A.CallTo(() => migrator_2_3.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => status.CompleteAsync(1)) A.CallTo(() => status.CompleteAsync(1))
@ -167,17 +168,17 @@ namespace Squidex.Infrastructure.Migrations
var sut = new Migrator(status, path, log); var sut = new Migrator(status, path, log);
A.CallTo(() => migrator_1_2.UpdateAsync()).Throws(new ArgumentException()); A.CallTo(() => migrator_1_2.UpdateAsync(A<CancellationToken>._)).Throws(new ArgumentException());
await Assert.ThrowsAsync<MigrationFailedException>(() => sut.MigrateAsync()); await Assert.ThrowsAsync<MigrationFailedException>(() => sut.MigrateAsync());
A.CallTo(() => migrator_0_1.UpdateAsync()) A.CallTo(() => migrator_0_1.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_1_2.UpdateAsync()) A.CallTo(() => migrator_1_2.UpdateAsync(A<CancellationToken>._))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_2_3.UpdateAsync()) A.CallTo(() => migrator_2_3.UpdateAsync(A<CancellationToken>._))
.MustNotHaveHappened(); .MustNotHaveHappened();
A.CallTo(() => status.CompleteAsync(1)) A.CallTo(() => status.CompleteAsync(1))
@ -201,7 +202,7 @@ namespace Squidex.Infrastructure.Migrations
var ex = new InvalidOperationException(); var ex = new InvalidOperationException();
A.CallTo(() => migrator_0_1.UpdateAsync()) A.CallTo(() => migrator_0_1.UpdateAsync(A<CancellationToken>._))
.Throws(ex); .Throws(ex);
var sut = new Migrator(status, path, log); var sut = new Migrator(status, path, log);
@ -211,7 +212,7 @@ namespace Squidex.Infrastructure.Migrations
A.CallTo(() => log.Log(SemanticLogLevel.Fatal, ex, A<LogFormatter>._!)) A.CallTo(() => log.Log(SemanticLogLevel.Fatal, ex, A<LogFormatter>._!))
.MustHaveHappened(); .MustHaveHappened();
A.CallTo(() => migrator_1_2.UpdateAsync()) A.CallTo(() => migrator_1_2.UpdateAsync(A<CancellationToken>._))
.MustNotHaveHappened(); .MustNotHaveHappened();
} }
@ -225,10 +226,10 @@ namespace Squidex.Infrastructure.Migrations
await Task.WhenAll(Enumerable.Repeat(0, 10).Select(x => Task.Run(() => sut.MigrateAsync()))); await Task.WhenAll(Enumerable.Repeat(0, 10).Select(x => Task.Run(() => sut.MigrateAsync())));
A.CallTo(() => migrator_0_1.UpdateAsync()) A.CallTo(() => migrator_0_1.UpdateAsync(A<CancellationToken>._))
.MustHaveHappenedOnceExactly(); .MustHaveHappenedOnceExactly();
A.CallTo(() => migrator_1_2.UpdateAsync()) A.CallTo(() => migrator_1_2.UpdateAsync(A<CancellationToken>._))
.MustHaveHappenedOnceExactly(); .MustHaveHappenedOnceExactly();
} }

Loading…
Cancel
Save