From ee7d6928923053bed41910ec95268d011aaf94e2 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 21 Jul 2018 11:46:25 +0200 Subject: [PATCH 01/23] Cleaner --- .../MongoAssetRepository_SnapshotStore.cs | 15 +- .../Contents/MongoContentCollection.cs | 5 + .../Contents/MongoContentDraftCollection.cs | 9 ++ .../Contents/MongoContentRepository.cs | 8 + .../MongoContentRepository_SnapshotStore.cs | 5 + .../Backup/CleanerGrain.cs | 147 ++++++++++++++++++ .../IAppStorage.cs | 17 ++ .../Rules/Indexes/IRulesByAppIndex.cs | 2 + .../Rules/Indexes/RulesByAppIndexGrain.cs | 5 + .../Schemas/Indexes/ISchemasByAppIndex.cs | 2 + .../Schemas/Indexes/SchemasByAppIndexGrain.cs | 5 + .../Tags/ITagGrain.cs | 2 + .../Tags/TagGrain.cs | 5 + .../EventSourcing/GetEventStore.cs | 10 ++ .../EventSourcing/MongoEventStore_Reader.cs | 14 +- .../EventSourcing/MongoEventStore_Writer.cs | 10 ++ .../States/MongoSnapshotStore.cs | 8 + .../EventSourcing/IEventStore.cs | 4 + .../States/IPersistence{TState}.cs | 2 + .../States/ISnapshotStore.cs | 2 + src/Squidex.Infrastructure/States/IStore.cs | 2 + .../States/Persistence{TSnapshot,TKey}.cs | 13 ++ src/Squidex.Infrastructure/States/Store.cs | 5 + 23 files changed, 285 insertions(+), 12 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs create mode 100644 src/Squidex.Domain.Apps.Entities/IAppStorage.cs diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs index 6cdbfecc6..189f08052 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs @@ -18,11 +18,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { public sealed partial class MongoAssetRepository : ISnapshotStore { - Task ISnapshotStore.ReadAllAsync(Func callback) - { - throw new NotSupportedException(); - } - public async Task<(AssetState Value, long Version)> ReadAsync(Guid key) { using (Profiler.TraceMethod()) @@ -52,5 +47,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets await Collection.ReplaceOneAsync(x => x.Id == key && x.Version == oldVersion, entity, Upsert); } } + + Task ISnapshotStore.ReadAllAsync(Func callback) + { + throw new NotSupportedException(); + } + + Task ISnapshotStore.RemoveAsync(Guid key) + { + throw new NotSupportedException(); + } } } diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs index ad6854d8b..eccc17915 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs @@ -117,5 +117,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents Filter.AnyNe(x => x.ReferencedIdsDeleted, id)), Update.AddToSet(x => x.ReferencedIdsDeleted, id)); } + + public Task RemoveAsync(Guid id) + { + return Collection.DeleteOneAsync(x => x.Id == id); + } } } diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs index 42a5050c9..ae894f6cf 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs @@ -58,6 +58,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return ids.Except(contentEntities.Select(x => Guid.Parse(x["_id"].AsString))).ToList(); } + public async Task> QueryIdsAsync(Guid appId) + { + var contentEntities = + await Collection.Find(x => x.IndexedAppId == appId).Only(x => x.Id) + .ToListAsync(); + + return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList(); + } + public Task QueryScheduledWithoutDataAsync(Instant now, Func callback) { return Collection.Find(x => x.ScheduledAt < now && x.IsDeleted != true) diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index 0d1f94f84..fb19628b3 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -99,6 +99,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } + public async Task> QueryIdsAsync(Guid appId) + { + using (Profiler.TraceMethod()) + { + return await contentsDraft.QueryIdsAsync(appId); + } + } + public async Task QueryScheduledWithoutDataAsync(Instant now, Func callback) { using (Profiler.TraceMethod()) diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs index 8393c28c3..cd6700635 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs @@ -83,6 +83,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return schema; } + Task ISnapshotStore.RemoveAsync(Guid key) + { + throw new NotSupportedException(); + } + Task ISnapshotStore.ReadAllAsync(Func callback) { throw new NotSupportedException(); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs new file mode 100644 index 000000000..a0db8d3fd --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs @@ -0,0 +1,147 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Orleans; +using Orleans.Concurrency; +using Orleans.Runtime; +using Squidex.Domain.Apps.Entities.Apps.State; +using Squidex.Domain.Apps.Entities.Rules; +using Squidex.Domain.Apps.Entities.Rules.State; +using Squidex.Domain.Apps.Entities.Schemas; +using Squidex.Domain.Apps.Entities.Schemas.State; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Orleans; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + [Reentrant] + public sealed class CleanerGrain : GrainOfGuid, IRemindable + { + private readonly IGrainFactory grainFactory; + private readonly IStore store; + private readonly IEventStore eventStore; + private readonly IEnumerable storages; + private IPersistence persistence; + private bool isCleaning; + private State state = new State(); + + [CollectionName("Index_AppsByName")] + public sealed class State + { + public HashSet Apps { get; set; } = new HashSet(); + + public HashSet PendingApps { get; set; } = new HashSet(); + } + + public CleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + Guard.NotNull(store, nameof(store)); + Guard.NotNull(storages, nameof(storages)); + Guard.NotNull(eventStore, nameof(eventStore)); + + this.grainFactory = grainFactory; + + this.store = store; + this.storages = storages; + + this.eventStore = eventStore; + } + + public async override Task OnActivateAsync(Guid key) + { + await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(10)); + + persistence = store.WithSnapshots(key, s => + { + state = s; + }); + + await persistence.ReadAsync(); + + await CleanAsync(); + } + + public Task ReceiveReminder(string reminderName, TickStatus status) + { + return CleanAsync(); + } + + private async Task CleanAsync() + { + if (isCleaning) + { + return; + } + + isCleaning = true; + try + { + foreach (var appId in state.Apps.ToList()) + { + try + { + await CleanAsync(appId); + + state.Apps.Remove(appId); + } + catch (NotSupportedException) + { + state.Apps.Remove(appId); + + state.PendingApps.Add(appId); + } + finally + { + await persistence.WriteSnapshotAsync(state); + } + } + } + finally + { + isCleaning = false; + } + } + + private async Task CleanAsync(Guid appId) + { + await eventStore.DeleteManyAsync("AppId", appId); + + var ruleIds = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); + + foreach (var ruleId in ruleIds) + { + await store.ClearSnapshotAsync(ruleId); + } + + var schemaIds = await grainFactory.GetGrain(appId).GetSchemaIdsAsync(); + + foreach (var schemaId in schemaIds) + { + await store.ClearSnapshotAsync(schemaId); + } + + foreach (var storage in storages) + { + await storage.ClearAsync(appId); + } + + await store.ClearSnapshotAsync(appId; + } + + private async Task DeleteAsync(Guid id) + { + await store.ClearSnapshotAsync(id); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/IAppStorage.cs b/src/Squidex.Domain.Apps.Entities/IAppStorage.cs new file mode 100644 index 000000000..16a051157 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/IAppStorage.cs @@ -0,0 +1,17 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; + +namespace Squidex.Domain.Apps.Entities +{ + public interface IAppStorage + { + Task ClearAsync(Guid appId); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs index dd169a045..a1a54459e 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs @@ -20,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Rules Task RebuildAsync(HashSet rules); + Task ClearAsync(); + Task> GetRuleIdsAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs index 6f9a49390..b3b87943d 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs @@ -69,5 +69,10 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes { return Task.FromResult(state.Rules.ToList()); } + + public Task ClearAsync() + { + return persistence.DeleteAsync(); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs index 38104d363..58b9f9e1f 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs @@ -18,6 +18,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas Task RemoveSchemaAsync(Guid schemaId); + Task ClearAsync(); + Task RebuildAsync(Dictionary schemas); Task GetSchemaIdAsync(string name); diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs index 9164936c9..cbec766f3 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs @@ -76,5 +76,10 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes { return Task.FromResult(state.Schemas.Values.ToList()); } + + public Task ClearAsync() + { + return persistence.DeleteAsync(); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs index a37bf83cb..2947aebda 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs @@ -20,5 +20,7 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> DenormalizeTagsAsync(HashSet ids); Task> GetTagsAsync(); + + Task ClearAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs index 3517b9476..66e17e6cb 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs @@ -147,5 +147,10 @@ namespace Squidex.Domain.Apps.Entities.Tags { return Task.FromResult(state.Tags.Values.ToDictionary(x => x.Name, x => x.Count)); } + + public Task ClearAsync() + { + return persistence.DeleteAsync(); + } } } diff --git a/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs b/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs index d21394747..50722a41d 100644 --- a/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs +++ b/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs @@ -118,6 +118,11 @@ namespace Squidex.Infrastructure.EventSourcing } } + public Task DeleteStreamAsync(string streamName) + { + return connection.DeleteStreamAsync(streamName, ExpectedVersion.Any); + } + public Task AppendAsync(Guid commitId, string streamName, ICollection events) { return AppendEventsInternalAsync(streamName, EtagVersion.Any, events); @@ -163,6 +168,11 @@ namespace Squidex.Infrastructure.EventSourcing } } + public Task DeleteManyAsync(string property, object value) + { + throw new NotSupportedException(); + } + private string GetStreamName(string streamName) { return $"{prefix}-{streamName}"; diff --git a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs index 9f7bc88f5..07861115d 100644 --- a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs +++ b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs @@ -124,8 +124,8 @@ namespace Squidex.Infrastructure.EventSourcing { var filters = new List>(); - AddPositionFilter(streamPosition, filters); - AddPropertyFitler(property, value, filters); + FilterByPosition(streamPosition, filters); + FilterByProperty(property, value, filters); return Filter.And(filters); } @@ -134,18 +134,18 @@ namespace Squidex.Infrastructure.EventSourcing { var filters = new List>(); - AddPositionFilter(streamPosition, filters); - AddStreamFilter(streamFilter, filters); + FilterByPosition(streamPosition, filters); + FilterByStream(streamFilter, filters); return Filter.And(filters); } - private static void AddPropertyFitler(string property, object value, List> filters) + private static void FilterByProperty(string property, object value, List> filters) { filters.Add(Filter.Eq(CreateIndexPath(property), value)); } - private static void AddStreamFilter(string streamFilter, List> filters) + private static void FilterByStream(string streamFilter, List> filters) { if (!string.IsNullOrWhiteSpace(streamFilter) && !string.Equals(streamFilter, ".*", StringComparison.OrdinalIgnoreCase)) { @@ -160,7 +160,7 @@ namespace Squidex.Infrastructure.EventSourcing } } - private static void AddPositionFilter(StreamPosition streamPosition, List> filters) + private static void FilterByPosition(StreamPosition streamPosition, List> filters) { if (streamPosition.IsEndOfCommit) { diff --git a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs index bbeb5c469..af4b0172a 100644 --- a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs +++ b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs @@ -20,6 +20,16 @@ namespace Squidex.Infrastructure.EventSourcing private const int MaxWriteAttempts = 20; private static readonly BsonTimestamp EmptyTimestamp = new BsonTimestamp(0); + public Task DeleteStreamAsync(string streamName) + { + return Collection.DeleteManyAsync(x => x.EventStream == streamName); + } + + public Task DeleteManyAsync(string property, object value) + { + return Collection.DeleteManyAsync(Filter.Eq(CreateIndexPath(property), value)); + } + public Task AppendAsync(Guid commitId, string streamName, ICollection events) { return AppendAsync(commitId, streamName, EtagVersion.Any, events); diff --git a/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs b/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs index 8fc6c3204..67e593b53 100644 --- a/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs @@ -68,5 +68,13 @@ namespace Squidex.Infrastructure.States await Collection.Find(new BsonDocument()).ForEachAsync(x => callback(x.Doc, x.Version)); } } + + public async Task RemoveAsync(TKey key) + { + using (Profiler.TraceMethod>()) + { + await Collection.DeleteOneAsync(x => x.Id.Equals(key)); + } + } } } diff --git a/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs b/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs index c33d86e5a..186b7310e 100644 --- a/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs +++ b/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs @@ -26,6 +26,10 @@ namespace Squidex.Infrastructure.EventSourcing Task AppendAsync(Guid commitId, string streamName, long expectedVersion, ICollection events); + Task DeleteStreamAsync(string streamName); + + Task DeleteManyAsync(string property, object value); + IEventSubscription CreateSubscription(IEventSubscriber subscriber, string streamFilter, string position = null); } } diff --git a/src/Squidex.Infrastructure/States/IPersistence{TState}.cs b/src/Squidex.Infrastructure/States/IPersistence{TState}.cs index 2b467b302..c69e95926 100644 --- a/src/Squidex.Infrastructure/States/IPersistence{TState}.cs +++ b/src/Squidex.Infrastructure/States/IPersistence{TState}.cs @@ -15,6 +15,8 @@ namespace Squidex.Infrastructure.States { long Version { get; } + Task DeleteAsync(); + Task WriteEventsAsync(IEnumerable> @events); Task WriteSnapshotAsync(TState state); diff --git a/src/Squidex.Infrastructure/States/ISnapshotStore.cs b/src/Squidex.Infrastructure/States/ISnapshotStore.cs index 60b2ab5be..9431226f1 100644 --- a/src/Squidex.Infrastructure/States/ISnapshotStore.cs +++ b/src/Squidex.Infrastructure/States/ISnapshotStore.cs @@ -18,6 +18,8 @@ namespace Squidex.Infrastructure.States Task ClearAsync(); + Task RemoveAsync(TKey key); + Task ReadAllAsync(Func callback); } } diff --git a/src/Squidex.Infrastructure/States/IStore.cs b/src/Squidex.Infrastructure/States/IStore.cs index e0c65ba83..8dee5d085 100644 --- a/src/Squidex.Infrastructure/States/IStore.cs +++ b/src/Squidex.Infrastructure/States/IStore.cs @@ -22,5 +22,7 @@ namespace Squidex.Infrastructure.States ISnapshotStore GetSnapshotStore(); Task ClearSnapshotsAsync(); + + Task ClearSnapshotAsync(TKey key); } } diff --git a/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs b/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs index ea8b50d1e..5c6a22c3f 100644 --- a/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs +++ b/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs @@ -175,6 +175,19 @@ namespace Squidex.Infrastructure.States UpdateVersion(); } + public async Task DeleteAsync() + { + if (UseEventSourcing()) + { + await eventStore.DeleteStreamAsync(GetStreamName()); + } + + if (UseSnapshots()) + { + await snapshotStore.RemoveAsync(ownerKey); + } + } + private EventData[] GetEventData(Envelope[] events, Guid commitId) { return @events.Select(x => eventDataFormatter.ToEventData(x, commitId, true)).ToArray(); diff --git a/src/Squidex.Infrastructure/States/Store.cs b/src/Squidex.Infrastructure/States/Store.cs index 0a3e02ff7..6f693c5a2 100644 --- a/src/Squidex.Infrastructure/States/Store.cs +++ b/src/Squidex.Infrastructure/States/Store.cs @@ -63,6 +63,11 @@ namespace Squidex.Infrastructure.States return GetSnapshotStore().ClearAsync(); } + public Task ClearSnapshotAsync(TKey key) + { + return GetSnapshotStore().RemoveAsync(key); + } + public ISnapshotStore GetSnapshotStore() { return (ISnapshotStore)services.GetService(typeof(ISnapshotStore)); From cb9a8ddb357ce6d681085bfb317ad4a4a2e22e3a Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 21 Jul 2018 19:53:27 +0200 Subject: [PATCH 02/23] Added some cleaners including tests. --- .../Backup/CleanerGrain.cs | 13 ++++-- .../Rules/Indexes/RuleIndexCleaner.cs | 31 ++++++++++++++ .../Rules/Indexes/RulesByAppIndexGrain.cs | 2 + .../Schemas/Indexes/SchemaIndexCleaner.cs | 31 ++++++++++++++ .../Schemas/Indexes/SchemasByAppIndexGrain.cs | 2 + .../AppsByNameIndexCommandMiddlewareTests.cs | 2 +- .../Apps/Indexes/AppsByNameIndexGrainTests.cs | 2 +- .../AppsByUserIndexCommandMiddlewareTests.cs | 2 +- .../Apps/Indexes/AppsByUserIndexGrainTests.cs | 2 +- .../SingletonCommandMiddlewareTests.cs | 2 +- .../Rules/Indexes/RuleIndexCleanerTests.cs | 40 +++++++++++++++++++ .../RulesByAppIndexCommandMiddlewareTests.cs | 11 +++-- .../Indexes/RulesByAppIndexGrainTests.cs | 17 +++++++- .../Indexes/SchemaIndexCleanerTests.cs | 40 +++++++++++++++++++ ...SchemasByAppIndexCommandMiddlewareTests.cs | 11 +++-- .../Indexes/SchemasByAppIndexGrainTests.cs | 16 +++++++- 16 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs create mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs create mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs index a0db8d3fd..8eb19922f 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs @@ -25,7 +25,7 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Backup { [Reentrant] - public sealed class CleanerGrain : GrainOfGuid, IRemindable + public sealed class CleanerGrain : GrainOfString, IRemindable, IBackgroundGrain { private readonly IGrainFactory grainFactory; private readonly IStore store; @@ -58,11 +58,11 @@ namespace Squidex.Domain.Apps.Entities.Backup this.eventStore = eventStore; } - public async override Task OnActivateAsync(Guid key) + public async override Task OnActivateAsync(string key) { await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(10)); - persistence = store.WithSnapshots(key, s => + persistence = store.WithSnapshots(Guid.Empty, s => { state = s; }); @@ -72,6 +72,11 @@ namespace Squidex.Domain.Apps.Entities.Backup await CleanAsync(); } + public Task ActivateAsync() + { + return CleanAsync(); + } + public Task ReceiveReminder(string reminderName, TickStatus status) { return CleanAsync(); @@ -136,7 +141,7 @@ namespace Squidex.Domain.Apps.Entities.Backup await storage.ClearAsync(appId); } - await store.ClearSnapshotAsync(appId; + await store.ClearSnapshotAsync(appId); } private async Task DeleteAsync(Guid id) diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs new file mode 100644 index 000000000..04af08f23 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Orleans; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Rules.Indexes +{ + public sealed class RuleIndexCleaner : IAppStorage + { + private readonly IGrainFactory grainFactory; + + public RuleIndexCleaner(IGrainFactory grainFactory) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + + this.grainFactory = grainFactory; + } + + public Task ClearAsync(Guid appId) + { + return grainFactory.GetGrain(appId).ClearAsync(); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs index b3b87943d..6d72c92d2 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs @@ -72,6 +72,8 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes public Task ClearAsync() { + state = new State(); + return persistence.DeleteAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs new file mode 100644 index 000000000..3945e7d1c --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Orleans; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Schemas.Indexes +{ + public sealed class SchemaIndexCleaner : IAppStorage + { + private readonly IGrainFactory grainFactory; + + public SchemaIndexCleaner(IGrainFactory grainFactory) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + + this.grainFactory = grainFactory; + } + + public Task ClearAsync(Guid appId) + { + return grainFactory.GetGrain(appId).ClearAsync(); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs index cbec766f3..57b743957 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs @@ -79,6 +79,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes public Task ClearAsync() { + state = new State(); + return persistence.DeleteAsync(); } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs index 8d67c852f..d01b8e2c9 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs @@ -16,7 +16,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Apps.Indexes { - public sealed class AppsByNameIndexCommandMiddlewareTests + public class AppsByNameIndexCommandMiddlewareTests { private readonly IGrainFactory grainFactory = A.Fake(); private readonly ICommandBus commandBus = A.Fake(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs index 5db614a21..7667f15a8 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs @@ -15,7 +15,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Apps.Indexes { - public sealed class AppsByNameIndexGrainTests + public class AppsByNameIndexGrainTests { private readonly IStore store = A.Fake>(); private readonly IPersistence persistence = A.Fake>(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexCommandMiddlewareTests.cs index afa3851b2..1f581b135 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexCommandMiddlewareTests.cs @@ -18,7 +18,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Apps.Indexes { - public sealed class AppsByUserIndexCommandMiddlewareTests + public class AppsByUserIndexCommandMiddlewareTests { private readonly IGrainFactory grainFactory = A.Fake(); private readonly ICommandBus commandBus = A.Fake(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexGrainTests.cs index f640a8831..ee8d91359 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByUserIndexGrainTests.cs @@ -15,7 +15,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Apps.Indexes { - public sealed class AppsByUserIndexGrainTests + public class AppsByUserIndexGrainTests { private readonly IStore store = A.Fake>(); private readonly IPersistence persistence = A.Fake>(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/SingletonCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/SingletonCommandMiddlewareTests.cs index 27587abd7..466c880c3 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/SingletonCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/SingletonCommandMiddlewareTests.cs @@ -14,7 +14,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Contents { - public sealed class SingletonCommandMiddlewareTests + public class SingletonCommandMiddlewareTests { private readonly ICommandBus commandBus = A.Fake(); private readonly SingletonCommandMiddleware sut = new SingletonCommandMiddleware(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs new file mode 100644 index 000000000..8d60f0d60 --- /dev/null +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs @@ -0,0 +1,40 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using FakeItEasy; +using Orleans; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Rules.Indexes +{ + public class RuleIndexCleanerTests + { + private readonly IGrainFactory grainFactory = A.Fake(); + private readonly IRulesByAppIndex index = A.Fake(); + private readonly Guid appId = Guid.NewGuid(); + private readonly RuleIndexCleaner sut; + + public RuleIndexCleanerTests() + { + A.CallTo(() => grainFactory.GetGrain(appId, null)) + .Returns(index); + + sut = new RuleIndexCleaner(grainFactory); + } + + [Fact] + public async Task Should_forward_to_index() + { + await sut.ClearAsync(appId); + + A.CallTo(() => index.ClearAsync()) + .MustHaveHappened(); + } + } +} diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexCommandMiddlewareTests.cs index cb4fd2a23..c90cc54a7 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexCommandMiddlewareTests.cs @@ -17,7 +17,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Rules.Indexes { - public sealed class RulesByAppIndexCommandMiddlewareTests + public class RulesByAppIndexCommandMiddlewareTests { private readonly IGrainFactory grainFactory = A.Fake(); private readonly ICommandBus commandBus = A.Fake(); @@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes public async Task Should_add_rule_to_index_on_create() { var context = - new CommandContext(new CreateRule { RuleId = appId, AppId = NamedId.Of(appId, "my-app") }, commandBus) + new CommandContext(new CreateRule { RuleId = appId, AppId = BuildAppId() }, commandBus) .Complete(); await sut.HandleAsync(context); @@ -60,7 +60,7 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes .Returns(J.AsTask(ruleState)); A.CallTo(() => ruleState.AppId) - .Returns(NamedId.Of(appId, "my-app")); + .Returns(BuildAppId()); var context = new CommandContext(new DeleteRule { RuleId = appId }, commandBus) @@ -71,5 +71,10 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes A.CallTo(() => index.RemoveRuleAsync(appId)) .MustHaveHappened(); } + + private NamedId BuildAppId() + { + return NamedId.Of(appId, "my-app"); + } } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexGrainTests.cs index be15f041f..98140a3e2 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RulesByAppIndexGrainTests.cs @@ -14,7 +14,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Rules.Indexes { - public sealed class RulesByAppIndexGrainTests + public class RulesByAppIndexGrainTests { private readonly IStore store = A.Fake>(); private readonly IPersistence persistence = A.Fake>(); @@ -45,6 +45,21 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes .MustHaveHappenedTwiceExactly(); } + [Fact] + public async Task Should_delete_and_reset_state_when_cleaning() + { + await sut.AddRuleAsync(ruleId1); + await sut.AddRuleAsync(ruleId2); + await sut.ClearAsync(); + + var ids = await sut.GetRuleIdsAsync(); + + Assert.Empty(ids); + + A.CallTo(() => persistence.DeleteAsync()) + .MustHaveHappened(); + } + [Fact] public async Task Should_remove_rule_id_from_index() { diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs new file mode 100644 index 000000000..721a5c21a --- /dev/null +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs @@ -0,0 +1,40 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using FakeItEasy; +using Orleans; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Schemas.Indexes +{ + public class SchemaIndexCleanerTests + { + private readonly IGrainFactory grainFactory = A.Fake(); + private readonly ISchemasByAppIndex index = A.Fake(); + private readonly Guid appId = Guid.NewGuid(); + private readonly SchemaIndexCleaner sut; + + public SchemaIndexCleanerTests() + { + A.CallTo(() => grainFactory.GetGrain(appId, null)) + .Returns(index); + + sut = new SchemaIndexCleaner(grainFactory); + } + + [Fact] + public async Task Should_forward_to_index() + { + await sut.ClearAsync(appId); + + A.CallTo(() => index.ClearAsync()) + .MustHaveHappened(); + } + } +} diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexCommandMiddlewareTests.cs index 15091edd8..d42b7e63d 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexCommandMiddlewareTests.cs @@ -17,7 +17,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Schemas.Indexes { - public sealed class SchemasByAppIndexCommandMiddlewareTests + public class SchemasByAppIndexCommandMiddlewareTests { private readonly IGrainFactory grainFactory = A.Fake(); private readonly ICommandBus commandBus = A.Fake(); @@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes public async Task Should_add_schema_to_index_on_create() { var context = - new CommandContext(new CreateSchema { SchemaId = appId, Name = "my-schema", AppId = NamedId.Of(appId, "my-app") }, commandBus) + new CommandContext(new CreateSchema { SchemaId = appId, Name = "my-schema", AppId = BuildAppId() }, commandBus) .Complete(); await sut.HandleAsync(context); @@ -60,7 +60,7 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes .Returns(J.AsTask(schemaState)); A.CallTo(() => schemaState.AppId) - .Returns(NamedId.Of(appId, "my-app")); + .Returns(BuildAppId()); var context = new CommandContext(new DeleteSchema { SchemaId = appId }, commandBus) @@ -71,5 +71,10 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes A.CallTo(() => index.RemoveSchemaAsync(appId)) .MustHaveHappened(); } + + private NamedId BuildAppId() + { + return NamedId.Of(appId, "my-app"); + } } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexGrainTests.cs index 5da814102..b52491a3a 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemasByAppIndexGrainTests.cs @@ -14,7 +14,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Schemas.Indexes { - public sealed class SchemasByAppIndexGrainTests + public class SchemasByAppIndexGrainTests { private readonly IStore store = A.Fake>(); private readonly IPersistence persistence = A.Fake>(); @@ -46,6 +46,20 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes .MustHaveHappened(); } + [Fact] + public async Task Should_delete_and_reset_state_when_cleaning() + { + await sut.AddSchemaAsync(schemaId1, schemaName1); + await sut.ClearAsync(); + + var id = await sut.GetSchemaIdAsync(schemaName1); + + Assert.Equal(id, Guid.Empty); + + A.CallTo(() => persistence.DeleteAsync()) + .MustHaveHappened(); + } + [Fact] public async Task Should_remove_schema_id_from_index() { From ed960c802745e326d08fd806861c86b1357e52f1 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 21 Jul 2018 20:04:16 +0200 Subject: [PATCH 03/23] More tests with states stores. --- .../Backup/CleanerGrain.cs | 8 +++--- .../Rules/Indexes/RulesByAppIndexGrain.cs | 10 +++---- .../Schemas/Indexes/SchemasByAppIndexGrain.cs | 10 +++---- src/Squidex.Infrastructure/States/IStore.cs | 2 +- src/Squidex.Infrastructure/States/Store.cs | 2 +- .../States/PersistenceEventSourcingTests.cs | 28 +++++++++++++++++++ .../States/PersistenceSnapshotTests.cs | 26 +++++++++++++++++ 7 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs index 8eb19922f..bd6b4c757 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs @@ -126,14 +126,14 @@ namespace Squidex.Domain.Apps.Entities.Backup foreach (var ruleId in ruleIds) { - await store.ClearSnapshotAsync(ruleId); + await store.RemoveSnapshotAsync(ruleId); } var schemaIds = await grainFactory.GetGrain(appId).GetSchemaIdsAsync(); foreach (var schemaId in schemaIds) { - await store.ClearSnapshotAsync(schemaId); + await store.RemoveSnapshotAsync(schemaId); } foreach (var storage in storages) @@ -141,12 +141,12 @@ namespace Squidex.Domain.Apps.Entities.Backup await storage.ClearAsync(appId); } - await store.ClearSnapshotAsync(appId); + await store.RemoveSnapshotAsync(appId); } private async Task DeleteAsync(Guid id) { - await store.ClearSnapshotAsync(id); + await store.RemoveSnapshotAsync(id); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs index 6d72c92d2..23be15dfa 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs @@ -65,16 +65,16 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes return persistence.WriteSnapshotAsync(state); } - public Task> GetRuleIdsAsync() - { - return Task.FromResult(state.Rules.ToList()); - } - public Task ClearAsync() { state = new State(); return persistence.DeleteAsync(); } + + public Task> GetRuleIdsAsync() + { + return Task.FromResult(state.Rules.ToList()); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs index 57b743957..dc2fc69c6 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs @@ -72,16 +72,16 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes return Task.FromResult(schemaId); } - public Task> GetSchemaIdsAsync() - { - return Task.FromResult(state.Schemas.Values.ToList()); - } - public Task ClearAsync() { state = new State(); return persistence.DeleteAsync(); } + + public Task> GetSchemaIdsAsync() + { + return Task.FromResult(state.Schemas.Values.ToList()); + } } } diff --git a/src/Squidex.Infrastructure/States/IStore.cs b/src/Squidex.Infrastructure/States/IStore.cs index 8dee5d085..f8aa2cddd 100644 --- a/src/Squidex.Infrastructure/States/IStore.cs +++ b/src/Squidex.Infrastructure/States/IStore.cs @@ -23,6 +23,6 @@ namespace Squidex.Infrastructure.States Task ClearSnapshotsAsync(); - Task ClearSnapshotAsync(TKey key); + Task RemoveSnapshotAsync(TKey key); } } diff --git a/src/Squidex.Infrastructure/States/Store.cs b/src/Squidex.Infrastructure/States/Store.cs index 6f693c5a2..3bbacc36d 100644 --- a/src/Squidex.Infrastructure/States/Store.cs +++ b/src/Squidex.Infrastructure/States/Store.cs @@ -63,7 +63,7 @@ namespace Squidex.Infrastructure.States return GetSnapshotStore().ClearAsync(); } - public Task ClearSnapshotAsync(TKey key) + public Task RemoveSnapshotAsync(TKey key) { return GetSnapshotStore().RemoveAsync(key); } diff --git a/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs b/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs index a910af1bf..3b23fb421 100644 --- a/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs +++ b/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs @@ -205,6 +205,34 @@ namespace Squidex.Infrastructure.States await Assert.ThrowsAsync(() => persistence.WriteEventsAsync(new[] { new MyEvent(), new MyEvent() }.Select(Envelope.Create))); } + [Fact] + public async Task Should_delete_events_but_not_snapshot_when_deleted_snapshot_only() + { + var persistence = sut.WithEventSourcing(key, x => { }); + + await persistence.DeleteAsync(); + + A.CallTo(() => eventStore.DeleteStreamAsync(key)) + .MustHaveHappened(); + + A.CallTo(() => snapshotStore.RemoveAsync(key)) + .MustNotHaveHappened(); + } + + [Fact] + public async Task Should_delete_events_and_snapshot_when_deleted() + { + var persistence = sut.WithSnapshotsAndEventSourcing(key, x => { }, x => { }); + + await persistence.DeleteAsync(); + + A.CallTo(() => eventStore.DeleteStreamAsync(key)) + .MustHaveHappened(); + + A.CallTo(() => snapshotStore.RemoveAsync(key)) + .MustHaveHappened(); + } + private void SetupEventStore(int count, int eventOffset = 0, int readPosition = 0) { SetupEventStore(Enumerable.Repeat(0, count).Select(x => new MyEvent()).ToArray(), eventOffset, readPosition); diff --git a/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs b/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs index b2e90f231..392f8efff 100644 --- a/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs +++ b/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs @@ -146,5 +146,31 @@ namespace Squidex.Infrastructure.States await Assert.ThrowsAsync(() => persistence.WriteSnapshotAsync(100)); } + + [Fact] + public async Task Should_delete_snapshot_but_not_events_when_deleted() + { + var persistence = sut.WithSnapshots(key, x => { }); + + await persistence.DeleteAsync(); + + A.CallTo(() => eventStore.DeleteStreamAsync(A.Ignored)) + .MustNotHaveHappened(); + + A.CallTo(() => snapshotStore.RemoveAsync(key)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_delete_snapshot_but_not_events_when_deleted_from_store() + { + await sut.RemoveSnapshotAsync(key); + + A.CallTo(() => eventStore.DeleteStreamAsync(A.Ignored)) + .MustNotHaveHappened(); + + A.CallTo(() => snapshotStore.RemoveAsync(key)) + .MustHaveHappened(); + } } } \ No newline at end of file From ee3afe88cbc79dabfadbd7bb1bcedb227c74b4e7 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 22 Jul 2018 17:04:38 +0200 Subject: [PATCH 04/23] Cleaner middleware, tests and service registration. --- .../Backup/CleanerGrain.cs | 9 ++- .../Backup/EnqueueAppToCleanerMiddleware.cs | 44 ++++++++++++++ .../Backup/ICleanerGrain.cs | 18 ++++++ .../Rules/Indexes/RulesByAppIndexGrain.cs | 14 ++--- .../Schemas/Indexes/SchemasByAppIndexGrain.cs | 14 ++--- .../Tags/GrainTagService.cs | 29 +++++---- .../Tags/ITagService.cs | 8 +-- .../Tags/TagGrain.cs | 12 ++-- src/Squidex/Config/Domain/EntitiesServices.cs | 60 ++++++++++++------- .../EnqueueAppToCleanerMiddlewareTests.cs | 48 +++++++++++++++ .../Tags/GrainTagServiceTests.cs | 11 +++- .../Tags/TagGrainTests.cs | 15 +++++ 12 files changed, 222 insertions(+), 60 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs create mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs index bd6b4c757..ffdeafa00 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs @@ -25,7 +25,7 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Backup { [Reentrant] - public sealed class CleanerGrain : GrainOfString, IRemindable, IBackgroundGrain + public sealed class CleanerGrain : GrainOfString, IRemindable, ICleanerGrain { private readonly IGrainFactory grainFactory; private readonly IStore store; @@ -72,6 +72,13 @@ namespace Squidex.Domain.Apps.Entities.Backup await CleanAsync(); } + public Task EnqueueAppAsync(Guid appId) + { + state.Apps.Add(appId); + + return persistence.WriteSnapshotAsync(state); + } + public Task ActivateAsync() { return CleanAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs new file mode 100644 index 000000000..3e6edfbcf --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs @@ -0,0 +1,44 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Orleans; +using Squidex.Domain.Apps.Entities.Apps.Commands; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Commands; +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public sealed class EnqueueAppToCleanerMiddleware : ICommandMiddleware + { + private readonly ICleanerGrain cleaner; + + public EnqueueAppToCleanerMiddleware(IGrainFactory grainFactory) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + + cleaner = grainFactory.GetGrain(SingleGrain.Id); + } + + public async Task HandleAsync(CommandContext context, Func next) + { + if (context.IsCompleted) + { + switch (context.Command) + { + case ArchiveApp archiveApp: + await cleaner.EnqueueAppAsync(archiveApp.AppId); + break; + } + } + + await next(); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs new file mode 100644 index 000000000..82077e895 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs @@ -0,0 +1,18 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface ICleanerGrain : IBackgroundGrain + { + Task EnqueueAppAsync(Guid appId); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs index 23be15dfa..911833a74 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs @@ -44,6 +44,13 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes return persistence.ReadAsync(); } + public Task ClearAsync() + { + state = new State(); + + return persistence.DeleteAsync(); + } + public Task RebuildAsync(HashSet rules) { state = new State { Rules = rules }; @@ -65,13 +72,6 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes return persistence.WriteSnapshotAsync(state); } - public Task ClearAsync() - { - state = new State(); - - return persistence.DeleteAsync(); - } - public Task> GetRuleIdsAsync() { return Task.FromResult(state.Rules.ToList()); diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs index dc2fc69c6..69eff9348 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs @@ -44,6 +44,13 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes return persistence.ReadAsync(); } + public Task ClearAsync() + { + state = new State(); + + return persistence.DeleteAsync(); + } + public Task RebuildAsync(Dictionary schemas) { state = new State { Schemas = schemas }; @@ -72,13 +79,6 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes return Task.FromResult(schemaId); } - public Task ClearAsync() - { - state = new State(); - - return persistence.DeleteAsync(); - } - public Task> GetSchemaIdsAsync() { return Task.FromResult(state.Schemas.Values.ToList()); diff --git a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs index ff869cf80..714759843 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs @@ -13,7 +13,7 @@ using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Tags { - public sealed class GrainTagService : ITagService + public sealed class GrainTagService : ITagService, IAppStorage { private readonly IGrainFactory grainFactory; @@ -24,31 +24,36 @@ namespace Squidex.Domain.Apps.Entities.Tags this.grainFactory = grainFactory; } - public Task> NormalizeTagsAsync(Guid appId, string category, HashSet names, HashSet ids) + public Task> NormalizeTagsAsync(Guid appId, string group, HashSet names, HashSet ids) { - return GetGrain(appId, category).NormalizeTagsAsync(names, ids); + return GetGrain(appId, group).NormalizeTagsAsync(names, ids); } - public Task> GetTagIdsAsync(Guid appId, string category, HashSet names) + public Task> GetTagIdsAsync(Guid appId, string group, HashSet names) { - return GetGrain(appId, category).GetTagIdsAsync(names); + return GetGrain(appId, group).GetTagIdsAsync(names); } - public Task> DenormalizeTagsAsync(Guid appId, string category, HashSet ids) + public Task> DenormalizeTagsAsync(Guid appId, string group, HashSet ids) { - return GetGrain(appId, category).DenormalizeTagsAsync(ids); + return GetGrain(appId, group).DenormalizeTagsAsync(ids); } - public Task> GetTagsAsync(Guid appId, string category) + public Task> GetTagsAsync(Guid appId, string group) { - return GetGrain(appId, category).GetTagsAsync(); + return GetGrain(appId, group).GetTagsAsync(); } - private ITagGrain GetGrain(Guid appId, string category) + public Task ClearAsync(Guid appId) { - Guard.NotNullOrEmpty(category, nameof(category)); + return GetGrain(appId, TagGroups.Assets).ClearAsync(); + } + + private ITagGrain GetGrain(Guid appId, string group) + { + Guard.NotNullOrEmpty(group, nameof(group)); - return grainFactory.GetGrain($"{appId}_{category}"); + return grainFactory.GetGrain($"{appId}_{group}"); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs index 52dae1dc4..b4972b99c 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs @@ -13,12 +13,12 @@ namespace Squidex.Domain.Apps.Entities.Tags { public interface ITagService { - Task> NormalizeTagsAsync(Guid appId, string category, HashSet names, HashSet ids); + Task> NormalizeTagsAsync(Guid appId, string group, HashSet names, HashSet ids); - Task> GetTagIdsAsync(Guid appId, string category, HashSet names); + Task> GetTagIdsAsync(Guid appId, string group, HashSet names); - Task> DenormalizeTagsAsync(Guid appId, string category, HashSet ids); + Task> DenormalizeTagsAsync(Guid appId, string group, HashSet ids); - Task> GetTagsAsync(Guid appId, string category); + Task> GetTagsAsync(Guid appId, string group); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs index 66e17e6cb..c0f9c6fac 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs @@ -51,6 +51,13 @@ namespace Squidex.Domain.Apps.Entities.Tags return persistence.ReadAsync(); } + public Task ClearAsync() + { + state = new State(); + + return persistence.DeleteAsync(); + } + public async Task> NormalizeTagsAsync(HashSet names, HashSet ids) { var result = new HashSet(); @@ -147,10 +154,5 @@ namespace Squidex.Domain.Apps.Entities.Tags { return Task.FromResult(state.Tags.Values.ToDictionary(x => x.Name, x => x.Count)); } - - public Task ClearAsync() - { - return persistence.DeleteAsync(); - } } } diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index c87313238..9f1da9a04 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -86,7 +86,7 @@ namespace Squidex.Config.Domain .AsSelf(); services.AddSingletonAs() - .As(); + .As().As(); services.AddSingletonAs() .As>(); @@ -94,6 +94,40 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As>(); + services.AddSingletonAs() + .As(); + + services.AddSingletonAs() + .As(); + + services.AddSingletonAs() + .As(); + + AddCommandPipeline(services); + + services.AddSingleton>(DomainObjectGrainFormatter.Format); + + services.AddSingleton(c => + { + var uiOptions = c.GetRequiredService>(); + + var result = new InitialPatterns(); + + foreach (var pattern in uiOptions.Value.RegexSuggestions) + { + if (!string.IsNullOrWhiteSpace(pattern.Key) && + !string.IsNullOrWhiteSpace(pattern.Value)) + { + result[Guid.NewGuid()] = new AppPattern(pattern.Key, pattern.Value); + } + } + + return result; + }); + } + + private static void AddCommandPipeline(IServiceCollection services) + { services.AddSingletonAs() .As(); @@ -151,28 +185,8 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); - services.AddSingletonAs() - .As(); - - services.AddSingleton>(DomainObjectGrainFormatter.Format); - - services.AddSingleton(c => - { - var uiOptions = c.GetRequiredService>(); - - var result = new InitialPatterns(); - - foreach (var pattern in uiOptions.Value.RegexSuggestions) - { - if (!string.IsNullOrWhiteSpace(pattern.Key) && - !string.IsNullOrWhiteSpace(pattern.Value)) - { - result[Guid.NewGuid()] = new AppPattern(pattern.Key, pattern.Value); - } - } - - return result; - }); + services.AddSingletonAs() + .As(); } public static void AddMyMigrationServices(this IServiceCollection services) diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs new file mode 100644 index 000000000..80819b22f --- /dev/null +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs @@ -0,0 +1,48 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using FakeItEasy; +using Orleans; +using Squidex.Domain.Apps.Entities.Apps.Commands; +using Squidex.Infrastructure.Commands; +using Squidex.Infrastructure.Orleans; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public class EnqueueAppToCleanerMiddlewareTests + { + private readonly IGrainFactory grainFactory = A.Fake(); + private readonly ICommandBus commandBus = A.Fake(); + private readonly ICleanerGrain index = A.Fake(); + private readonly Guid appId = Guid.NewGuid(); + private readonly EnqueueAppToCleanerMiddleware sut; + + public EnqueueAppToCleanerMiddlewareTests() + { + A.CallTo(() => grainFactory.GetGrain(SingleGrain.Id, null)) + .Returns(index); + + sut = new EnqueueAppToCleanerMiddleware(grainFactory); + } + + [Fact] + public async Task Should_enqueue_for_cleanup_on_archive() + { + var context = + new CommandContext(new ArchiveApp { AppId = appId }, commandBus) + .Complete(); + + await sut.HandleAsync(context); + + A.CallTo(() => index.EnqueueAppAsync(appId)) + .MustHaveHappened(); + } + } +} diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs index f1aa0fd33..64cb5eebe 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs @@ -30,7 +30,16 @@ namespace Squidex.Domain.Apps.Entities.Tags } [Fact] - public async Task Should_call_grain_when_retrieving_tas() + public async Task Should_call_grain_when_clearing() + { + await sut.ClearAsync(appId); + + A.CallTo(() => grain.ClearAsync()) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_call_grain_when_retrieving_tags() { await sut.GetTagsAsync(appId, TagGroups.Assets); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs index ce385f017..998252ada 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs @@ -30,6 +30,21 @@ namespace Squidex.Domain.Apps.Entities.Tags sut.OnActivateAsync(string.Empty).Wait(); } + [Fact] + public async Task Should_delete_and_reset_state_when_cleaning() + { + await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2"), null); + await sut.NormalizeTagsAsync(HashSet.Of("tag2", "tag3"), null); + await sut.ClearAsync(); + + var allTags = await sut.GetTagsAsync(); + + Assert.Empty(allTags); + + A.CallTo(() => persistence.DeleteAsync()) + .MustHaveHappened(); + } + [Fact] public async Task Should_add_tags_to_grain() { From 63fa6b8b028d0a9ea4c015764031acb039be03aa Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 22 Jul 2018 17:56:43 +0200 Subject: [PATCH 05/23] Unified some cleaners. --- ...hemaIndexCleaner.cs => AppGrainCleaner.cs} | 8 ++-- .../{CleanerGrain.cs => AppCleanerGrain.cs} | 19 +++++---- .../Backup/BackupGrain.cs | 13 ++++++ .../Backup/EnqueueAppToCleanerMiddleware.cs | 4 +- .../{ICleanerGrain.cs => IAppCleanerGrain.cs} | 2 +- .../Backup/IBackupGrain.cs | 3 +- .../ICleanableAppGrain.cs | 17 ++++++++ ...IAppStorage.cs => ICleanableAppStorage.cs} | 2 +- .../Rules/Indexes/IRulesByAppIndex.cs | 5 +-- .../Rules/Indexes/RuleIndexCleaner.cs | 31 -------------- .../Schemas/Indexes/ISchemasByAppIndex.cs | 5 +-- .../Tags/GrainTagService.cs | 2 +- src/Squidex/Config/Domain/EntitiesServices.cs | 16 ++++---- src/Squidex/Config/Orleans/SiloWrapper.cs | 2 + ...leanerTests.cs => AppGrainCleanerTests.cs} | 14 +++---- .../EnqueueAppToCleanerMiddlewareTests.cs | 4 +- .../Indexes/SchemaIndexCleanerTests.cs | 40 ------------------- 17 files changed, 74 insertions(+), 113 deletions(-) rename src/Squidex.Domain.Apps.Entities/{Schemas/Indexes/SchemaIndexCleaner.cs => AppGrainCleaner.cs} (73%) rename src/Squidex.Domain.Apps.Entities/Backup/{CleanerGrain.cs => AppCleanerGrain.cs} (87%) rename src/Squidex.Domain.Apps.Entities/Backup/{ICleanerGrain.cs => IAppCleanerGrain.cs} (90%) create mode 100644 src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs rename src/Squidex.Domain.Apps.Entities/{IAppStorage.cs => ICleanableAppStorage.cs} (92%) delete mode 100644 src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs rename tests/Squidex.Domain.Apps.Entities.Tests/{Rules/Indexes/RuleIndexCleanerTests.cs => AppGrainCleanerTests.cs} (67%) delete mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs b/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs similarity index 73% rename from src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs rename to src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs index 3945e7d1c..1d777f3e1 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemaIndexCleaner.cs +++ b/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs @@ -10,13 +10,13 @@ using System.Threading.Tasks; using Orleans; using Squidex.Infrastructure; -namespace Squidex.Domain.Apps.Entities.Schemas.Indexes +namespace Squidex.Domain.Apps.Entities { - public sealed class SchemaIndexCleaner : IAppStorage + public sealed class AppGrainCleaner : ICleanableAppStorage where T : ICleanableAppGrain { private readonly IGrainFactory grainFactory; - public SchemaIndexCleaner(IGrainFactory grainFactory) + public AppGrainCleaner(IGrainFactory grainFactory) { Guard.NotNull(grainFactory, nameof(grainFactory)); @@ -25,7 +25,7 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes public Task ClearAsync(Guid appId) { - return grainFactory.GetGrain(appId).ClearAsync(); + return grainFactory.GetGrain(appId).ClearAsync(); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs similarity index 87% rename from src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs rename to src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs index ffdeafa00..d5724e662 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs @@ -21,16 +21,17 @@ using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Backup { [Reentrant] - public sealed class CleanerGrain : GrainOfString, IRemindable, ICleanerGrain + public sealed class AppCleanerGrain : GrainOfString, IRemindable, IAppCleanerGrain { private readonly IGrainFactory grainFactory; private readonly IStore store; private readonly IEventStore eventStore; - private readonly IEnumerable storages; + private readonly IEnumerable storages; private IPersistence persistence; private bool isCleaning; private State state = new State(); @@ -43,7 +44,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public HashSet PendingApps { get; set; } = new HashSet(); } - public CleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages) + public AppCleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages) { Guard.NotNull(grainFactory, nameof(grainFactory)); Guard.NotNull(store, nameof(store)); @@ -60,9 +61,9 @@ namespace Squidex.Domain.Apps.Entities.Backup public async override Task OnActivateAsync(string key) { - await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(10)); + await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(2)); - persistence = store.WithSnapshots(Guid.Empty, s => + persistence = store.WithSnapshots(Guid.Empty, s => { state = s; }); @@ -81,12 +82,16 @@ namespace Squidex.Domain.Apps.Entities.Backup public Task ActivateAsync() { - return CleanAsync(); + CleanAsync().Forget(); + + return TaskHelper.Done; } public Task ReceiveReminder(string reminderName, TickStatus status) { - return CleanAsync(); + CleanAsync().Forget(); + + return TaskHelper.Done; } private async Task CleanAsync() diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index f1d0a8882..6aa82decd 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -84,6 +84,19 @@ namespace Squidex.Domain.Apps.Entities.Backup await CleanupAsync(); } + public async Task ClearAsync() + { + foreach (var job in state.Jobs) + { + await CleanupArchiveAsync(job); + await CleanupBackupAsync(job); + } + + state = new BackupState(); + + await persistence.DeleteAsync(); + } + private async Task ReadAsync() { await persistence.ReadAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs index 3e6edfbcf..2d479892b 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs @@ -17,13 +17,13 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class EnqueueAppToCleanerMiddleware : ICommandMiddleware { - private readonly ICleanerGrain cleaner; + private readonly IAppCleanerGrain cleaner; public EnqueueAppToCleanerMiddleware(IGrainFactory grainFactory) { Guard.NotNull(grainFactory, nameof(grainFactory)); - cleaner = grainFactory.GetGrain(SingleGrain.Id); + cleaner = grainFactory.GetGrain(SingleGrain.Id); } public async Task HandleAsync(CommandContext context, Func next) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs similarity index 90% rename from src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs rename to src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs index 82077e895..0059a4629 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/ICleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs @@ -11,7 +11,7 @@ using Squidex.Infrastructure.Orleans; namespace Squidex.Domain.Apps.Entities.Backup { - public interface ICleanerGrain : IBackgroundGrain + public interface IAppCleanerGrain : IBackgroundGrain { Task EnqueueAppAsync(Guid appId); } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs index 21f66e2ff..ec545f3dd 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs @@ -8,12 +8,11 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Orleans; using Squidex.Infrastructure.Orleans; namespace Squidex.Domain.Apps.Entities.Backup { - public interface IBackupGrain : IGrainWithGuidKey + public interface IBackupGrain : ICleanableAppGrain { Task RunAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs b/src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs new file mode 100644 index 000000000..4c9d97206 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs @@ -0,0 +1,17 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Orleans; + +namespace Squidex.Domain.Apps.Entities +{ + public interface ICleanableAppGrain : IGrainWithGuidKey + { + Task ClearAsync(); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/IAppStorage.cs b/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs similarity index 92% rename from src/Squidex.Domain.Apps.Entities/IAppStorage.cs rename to src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs index 16a051157..244388906 100644 --- a/src/Squidex.Domain.Apps.Entities/IAppStorage.cs +++ b/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs @@ -10,7 +10,7 @@ using System.Threading.Tasks; namespace Squidex.Domain.Apps.Entities { - public interface IAppStorage + public interface ICleanableAppStorage { Task ClearAsync(Guid appId); } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs index a1a54459e..00fc680b0 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs @@ -8,11 +8,10 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Orleans; namespace Squidex.Domain.Apps.Entities.Rules { - public interface IRulesByAppIndex : IGrainWithGuidKey + public interface IRulesByAppIndex : ICleanableAppGrain { Task AddRuleAsync(Guid ruleId); @@ -20,8 +19,6 @@ namespace Squidex.Domain.Apps.Entities.Rules Task RebuildAsync(HashSet rules); - Task ClearAsync(); - Task> GetRuleIdsAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs deleted file mode 100644 index 04af08f23..000000000 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RuleIndexCleaner.cs +++ /dev/null @@ -1,31 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using Orleans; -using Squidex.Infrastructure; - -namespace Squidex.Domain.Apps.Entities.Rules.Indexes -{ - public sealed class RuleIndexCleaner : IAppStorage - { - private readonly IGrainFactory grainFactory; - - public RuleIndexCleaner(IGrainFactory grainFactory) - { - Guard.NotNull(grainFactory, nameof(grainFactory)); - - this.grainFactory = grainFactory; - } - - public Task ClearAsync(Guid appId) - { - return grainFactory.GetGrain(appId).ClearAsync(); - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs index 58b9f9e1f..17da42a4e 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs @@ -8,18 +8,15 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Orleans; namespace Squidex.Domain.Apps.Entities.Schemas { - public interface ISchemasByAppIndex : IGrainWithGuidKey + public interface ISchemasByAppIndex : ICleanableAppGrain { Task AddSchemaAsync(Guid schemaId, string name); Task RemoveSchemaAsync(Guid schemaId); - Task ClearAsync(); - Task RebuildAsync(Dictionary schemas); Task GetSchemaIdAsync(string name); diff --git a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs index 714759843..5eafa38e2 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs @@ -13,7 +13,7 @@ using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Tags { - public sealed class GrainTagService : ITagService, IAppStorage + public sealed class GrainTagService : ITagService, ICleanableAppStorage { private readonly IGrainFactory grainFactory; diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index 9f1da9a04..5d58210c4 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -52,8 +52,7 @@ namespace Squidex.Config.Domain c.GetRequiredService>(), c.GetRequiredService(), exposeSourceUrl)) - .As() - .As(); + .As().As(); services.AddSingletonAs() .As(); @@ -86,7 +85,7 @@ namespace Squidex.Config.Domain .AsSelf(); services.AddSingletonAs() - .As().As(); + .As().As(); services.AddSingletonAs() .As>(); @@ -94,11 +93,14 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As>(); - services.AddSingletonAs() - .As(); + services.AddSingletonAs>() + .As(); - services.AddSingletonAs() - .As(); + services.AddSingletonAs>() + .As(); + + services.AddSingletonAs>() + .As(); services.AddSingletonAs() .As(); diff --git a/src/Squidex/Config/Orleans/SiloWrapper.cs b/src/Squidex/Config/Orleans/SiloWrapper.cs index 62139bb06..38292f58c 100644 --- a/src/Squidex/Config/Orleans/SiloWrapper.cs +++ b/src/Squidex/Config/Orleans/SiloWrapper.cs @@ -15,6 +15,7 @@ using Orleans; using Orleans.Configuration; using Orleans.Hosting; using Squidex.Config.Domain; +using Squidex.Domain.Apps.Entities.Backup; using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Rules; using Squidex.Infrastructure; @@ -62,6 +63,7 @@ namespace Squidex.Config.Orleans .EnableDirectClient() .AddIncomingGrainCallFilter() .AddStartupTask>() + .AddStartupTask>() .AddStartupTask>() .AddStartupTask>() .AddStartupTask((services, ct) => diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs similarity index 67% rename from tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs rename to tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs index 8d60f0d60..db2ed08d1 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Rules/Indexes/RuleIndexCleanerTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs @@ -11,21 +11,21 @@ using FakeItEasy; using Orleans; using Xunit; -namespace Squidex.Domain.Apps.Entities.Rules.Indexes +namespace Squidex.Domain.Apps.Entities { - public class RuleIndexCleanerTests + public class AppGrainCleanerTests { private readonly IGrainFactory grainFactory = A.Fake(); - private readonly IRulesByAppIndex index = A.Fake(); + private readonly ICleanableAppGrain index = A.Fake(); private readonly Guid appId = Guid.NewGuid(); - private readonly RuleIndexCleaner sut; + private readonly AppGrainCleaner sut; - public RuleIndexCleanerTests() + public AppGrainCleanerTests() { - A.CallTo(() => grainFactory.GetGrain(appId, null)) + A.CallTo(() => grainFactory.GetGrain(appId, null)) .Returns(index); - sut = new RuleIndexCleaner(grainFactory); + sut = new AppGrainCleaner(grainFactory); } [Fact] diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs index 80819b22f..0a8c24350 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs @@ -20,13 +20,13 @@ namespace Squidex.Domain.Apps.Entities.Backup { private readonly IGrainFactory grainFactory = A.Fake(); private readonly ICommandBus commandBus = A.Fake(); - private readonly ICleanerGrain index = A.Fake(); + private readonly IAppCleanerGrain index = A.Fake(); private readonly Guid appId = Guid.NewGuid(); private readonly EnqueueAppToCleanerMiddleware sut; public EnqueueAppToCleanerMiddlewareTests() { - A.CallTo(() => grainFactory.GetGrain(SingleGrain.Id, null)) + A.CallTo(() => grainFactory.GetGrain(SingleGrain.Id, null)) .Returns(index); sut = new EnqueueAppToCleanerMiddleware(grainFactory); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs deleted file mode 100644 index 721a5c21a..000000000 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/Indexes/SchemaIndexCleanerTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using FakeItEasy; -using Orleans; -using Xunit; - -namespace Squidex.Domain.Apps.Entities.Schemas.Indexes -{ - public class SchemaIndexCleanerTests - { - private readonly IGrainFactory grainFactory = A.Fake(); - private readonly ISchemasByAppIndex index = A.Fake(); - private readonly Guid appId = Guid.NewGuid(); - private readonly SchemaIndexCleaner sut; - - public SchemaIndexCleanerTests() - { - A.CallTo(() => grainFactory.GetGrain(appId, null)) - .Returns(index); - - sut = new SchemaIndexCleaner(grainFactory); - } - - [Fact] - public async Task Should_forward_to_index() - { - await sut.ClearAsync(appId); - - A.CallTo(() => index.ClearAsync()) - .MustHaveHappened(); - } - } -} From 950ae74f45e02756cc765b59dcc56dbf0d2277e4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 23 Jul 2018 17:55:48 +0200 Subject: [PATCH 06/23] AssetAlreadyExistsException.cs for Folder and GridFS --- .../Assets/AzureBlobAssetStore.cs | 4 +- .../Assets/GoogleCloudAssetStore.cs | 4 +- .../Assets/MongoGridFsAssetStore.cs | 34 +++++--- .../Assets/AssetAlreadyExistsException.cs | 38 +++++++++ .../Assets/AssetNotFoundException.cs | 17 ++-- .../Assets/FolderAssetStore.cs | 78 +++++++++++-------- .../Assets/IAssetStore.cs | 6 +- .../Assets/AssetStoreTests.cs | 62 +++++++-------- 8 files changed, 152 insertions(+), 91 deletions(-) create mode 100644 src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs diff --git a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs index c5f841510..6daa79886 100644 --- a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs +++ b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs @@ -83,7 +83,7 @@ namespace Squidex.Infrastructure.Assets } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404) { - throw new AssetNotFoundException($"Asset {name} not found.", ex); + throw new AssetNotFoundException(name, ex); } } @@ -98,7 +98,7 @@ namespace Squidex.Infrastructure.Assets } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404) { - throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); + throw new AssetNotFoundException($"Id={id}, Version={version}", ex); } } diff --git a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs index 5af2be27a..7251d070a 100644 --- a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs +++ b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs @@ -71,7 +71,7 @@ namespace Squidex.Infrastructure.Assets } catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound) { - throw new AssetNotFoundException($"Asset {name} not found.", ex); + throw new AssetNotFoundException(name, ex); } } @@ -85,7 +85,7 @@ namespace Squidex.Infrastructure.Assets } catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound) { - throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); + throw new AssetNotFoundException($"Id={id}, Version={version}", ex); } } diff --git a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs index 253641cd7..dbc093b37 100644 --- a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs @@ -9,6 +9,7 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.GridFS; @@ -43,20 +44,20 @@ namespace Squidex.Infrastructure.Assets return "UNSUPPORTED"; } - public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) + public async Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { try { var target = GetFileName(id, version, suffix); - using (var readStream = await bucket.OpenDownloadStreamAsync(name, cancellationToken: ct)) + using (var readStream = await bucket.OpenDownloadStreamAsync(sourceFileName, cancellationToken: ct)) { - await bucket.UploadFromStreamAsync(target, target, readStream, cancellationToken: ct); + await UploadFileCoreAsync(target, readStream, ct); } } catch (GridFSFileNotFoundException ex) { - throw new AssetNotFoundException($"Asset {name} not found.", ex); + throw new AssetNotFoundException(sourceFileName, ex); } } @@ -73,13 +74,13 @@ namespace Squidex.Infrastructure.Assets } catch (GridFSFileNotFoundException ex) { - throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); + throw new AssetNotFoundException($"Id={id}, Version={version}", ex); } } - public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) { - return UploadFileCoreAsync(name, stream, ct); + return UploadFileCoreAsync(fileName, stream, ct); } public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) @@ -87,9 +88,9 @@ namespace Squidex.Infrastructure.Assets return UploadFileCoreAsync(GetFileName(id, version, suffix), stream, ct); } - public Task DeleteAsync(string name) + public Task DeleteAsync(string fileName) { - return DeleteCoreAsync(name); + return DeleteCoreAsync(fileName); } public Task DeleteAsync(string id, long version, string suffix) @@ -109,9 +110,20 @@ namespace Squidex.Infrastructure.Assets } } - private Task UploadFileCoreAsync(string id, Stream stream, CancellationToken ct = default(CancellationToken)) + private async Task UploadFileCoreAsync(string id, Stream stream, CancellationToken ct = default(CancellationToken)) { - return bucket.UploadFromStreamAsync(id, id, stream, cancellationToken: ct); + try + { + await bucket.UploadFromStreamAsync(id, id, stream, cancellationToken: ct); + } + catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + throw new AssetAlreadyExistsException(id); + } + catch (MongoBulkWriteException ex) when (ex.WriteErrors.Any(x => x.Category == ServerErrorCategory.DuplicateKey)) + { + throw new AssetAlreadyExistsException(id); + } } private static string GetFileName(string id, long version, string suffix) diff --git a/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs b/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs new file mode 100644 index 000000000..954f26c4c --- /dev/null +++ b/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs @@ -0,0 +1,38 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschränkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Runtime.Serialization; + +namespace Squidex.Infrastructure.Assets +{ + [Serializable] + public class AssetAlreadyExistsException : Exception + { + public AssetAlreadyExistsException(string fileName) + : base(FormatMessage(fileName)) + { + } + + public AssetAlreadyExistsException(string fileName, Exception inner) + : base(FormatMessage(fileName), inner) + { + } + + protected AssetAlreadyExistsException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + + private static string FormatMessage(string fileName) + { + Guard.NotNullOrEmpty(fileName, nameof(fileName)); + + return $"An asset with name '{fileName}' already not exists."; + } + } +} diff --git a/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs b/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs index 104371efb..1691a8bbf 100644 --- a/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs +++ b/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs @@ -13,23 +13,26 @@ namespace Squidex.Infrastructure.Assets [Serializable] public class AssetNotFoundException : Exception { - public AssetNotFoundException() + public AssetNotFoundException(string fileName) + : base(FormatMessage(fileName)) { } - public AssetNotFoundException(string message) - : base(message) + public AssetNotFoundException(string fileName, Exception inner) + : base(FormatMessage(fileName), inner) { } - public AssetNotFoundException(string message, Exception inner) - : base(message, inner) + protected AssetNotFoundException(SerializationInfo info, StreamingContext context) + : base(info, context) { } - protected AssetNotFoundException(SerializationInfo info, StreamingContext context) - : base(info, context) + private static string FormatMessage(string fileName) { + Guard.NotNullOrEmpty(fileName, nameof(fileName)); + + return $"An asset with name '{fileName}' does not exist."; } } } diff --git a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs index 6ff9d8598..9e3772530 100644 --- a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs @@ -59,26 +59,6 @@ namespace Squidex.Infrastructure.Assets return file.FullName; } - public async Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) - { - var file = GetFile(name); - - using (var fileStream = file.OpenWrite()) - { - await stream.CopyToAsync(fileStream, BufferSize, ct); - } - } - - public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) - { - var file = GetFile(id, version, suffix); - - using (var fileStream = file.OpenWrite()) - { - await stream.CopyToAsync(fileStream, BufferSize, ct); - } - } - public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var file = GetFile(id, version, suffix); @@ -92,44 +72,74 @@ namespace Squidex.Infrastructure.Assets } catch (FileNotFoundException ex) { - throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); + throw new AssetNotFoundException($"Id={id}, Version={version}", ex); } } - public Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) + public Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { + var targetFile = GetFile(id, version, suffix); + try { - var file = GetFile(name); + var file = GetFile(sourceFileName); - file.CopyTo(GetPath(id, version, suffix)); + file.CopyTo(targetFile.FullName); return TaskHelper.Done; } + catch (IOException) when (targetFile.Exists) + { + throw new AssetAlreadyExistsException(targetFile.Name); + } catch (FileNotFoundException ex) { - throw new AssetNotFoundException($"Asset {name} not found.", ex); + throw new AssetNotFoundException(sourceFileName, ex); } } - public Task DeleteAsync(string id, long version, string suffix) + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) { - var file = GetFile(id, version, suffix); + return UploadCoreAsync(GetFile(fileName), stream, ct); + } - file.Delete(); + public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) + { + return UploadCoreAsync(GetFile(id, version, suffix), stream, ct); + } - return TaskHelper.Done; + public Task DeleteAsync(string id, long version, string suffix) + { + return DeleteFileCoreAsync(GetFile(id, version, suffix)); } - public Task DeleteAsync(string name) + public Task DeleteAsync(string fileName) { - var file = GetFile(name); + return DeleteFileCoreAsync(GetFile(fileName)); + } + private static Task DeleteFileCoreAsync(FileInfo file) + { file.Delete(); return TaskHelper.Done; } + private async Task UploadCoreAsync(FileInfo file, Stream stream, CancellationToken ct) + { + try + { + using (var fileStream = file.Open(FileMode.CreateNew, FileAccess.Write)) + { + await stream.CopyToAsync(fileStream, BufferSize, ct); + } + } + catch (IOException) when (file.Exists) + { + throw new AssetAlreadyExistsException(file.Name); + } + } + private FileInfo GetFile(string id, long version, string suffix) { Guard.NotNullOrEmpty(id, nameof(id)); @@ -137,11 +147,11 @@ namespace Squidex.Infrastructure.Assets return GetFile(GetPath(id, version, suffix)); } - private FileInfo GetFile(string name) + private FileInfo GetFile(string fileName) { - Guard.NotNullOrEmpty(name, nameof(name)); + Guard.NotNullOrEmpty(fileName, nameof(fileName)); - return new FileInfo(GetPath(name)); + return new FileInfo(GetPath(fileName)); } private string GetPath(string name) diff --git a/src/Squidex.Infrastructure/Assets/IAssetStore.cs b/src/Squidex.Infrastructure/Assets/IAssetStore.cs index 7a4d2cdbc..8f954d731 100644 --- a/src/Squidex.Infrastructure/Assets/IAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/IAssetStore.cs @@ -15,15 +15,15 @@ namespace Squidex.Infrastructure.Assets { string GenerateSourceUrl(string id, long version, string suffix); - Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)); + Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)); Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)); - Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)); + Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)); Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)); - Task DeleteAsync(string name); + Task DeleteAsync(string fileName); Task DeleteAsync(string id, long version, string suffix); } diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index cf5d56896..e384b4b34 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -14,11 +14,16 @@ namespace Squidex.Infrastructure.Assets { public abstract class AssetStoreTests : IDisposable where T : IAssetStore { + private readonly MemoryStream assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); + private readonly string assetId = Guid.NewGuid().ToString(); + private readonly string tempId = Guid.NewGuid().ToString(); private readonly Lazy sut; protected AssetStoreTests() { sut = new Lazy(CreateStore); + + ((IInitializable)Sut).Initialize(); } protected T Sut @@ -33,27 +38,18 @@ namespace Squidex.Infrastructure.Assets [Fact] public virtual Task Should_throw_exception_if_asset_to_download_is_not_found() { - ((IInitializable)Sut).Initialize(); - - return Assert.ThrowsAsync(() => Sut.DownloadAsync(Id(), 1, "suffix", new MemoryStream())); + return Assert.ThrowsAsync(() => Sut.DownloadAsync(assetId, 1, "suffix", new MemoryStream())); } [Fact] public Task Should_throw_exception_if_asset_to_copy_is_not_found() { - ((IInitializable)Sut).Initialize(); - - return Assert.ThrowsAsync(() => Sut.CopyAsync(Id(), Id(), 1, null)); + return Assert.ThrowsAsync(() => Sut.CopyAsync(tempId, assetId, 1, null)); } [Fact] public async Task Should_read_and_write_file() { - ((IInitializable)Sut).Initialize(); - - var assetId = Id(); - var assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); - await Sut.UploadAsync(assetId, 1, "suffix", assetData); var readData = new MemoryStream(); @@ -64,15 +60,16 @@ namespace Squidex.Infrastructure.Assets } [Fact] - public virtual async Task Should_commit_temporary_file() + public async Task Should_throw_exception_when_file_to_write_already_exists() { - ((IInitializable)Sut).Initialize(); - - var tempId = Id(); + await Sut.UploadAsync(assetId, 1, "suffix", assetData); - var assetId = Id(); - var assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); + await Assert.ThrowsAsync(() => Sut.UploadAsync(assetId, 1, "suffix", assetData)); + } + [Fact] + public virtual async Task Should_read_and_write_temporary_file() + { await Sut.UploadAsync(tempId, assetData); await Sut.CopyAsync(tempId, assetId, 1, "suffix"); @@ -84,14 +81,26 @@ namespace Squidex.Infrastructure.Assets } [Fact] - public async Task Should_ignore_when_deleting_twice_by_name() + public async Task Should_throw_exception_when_temporary_file_to_write_already_exists() { - ((IInitializable)Sut).Initialize(); + await Sut.UploadAsync(tempId, assetData); + await Sut.CopyAsync(tempId, assetId, 1, "suffix"); - var tempId = Id(); + await Assert.ThrowsAsync(() => Sut.UploadAsync(tempId, assetData)); + } - var assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); + [Fact] + public async Task Should_throw_exception_when_target_file_to_copy_to_already_exists() + { + await Sut.UploadAsync(tempId, assetData); + await Sut.CopyAsync(tempId, assetId, 1, "suffix"); + + await Assert.ThrowsAsync(() => Sut.CopyAsync(tempId, assetId, 1, "suffix")); + } + [Fact] + public async Task Should_ignore_when_deleting_twice_by_name() + { await Sut.UploadAsync(tempId, assetData); await Sut.DeleteAsync(tempId); await Sut.DeleteAsync(tempId); @@ -100,20 +109,9 @@ namespace Squidex.Infrastructure.Assets [Fact] public async Task Should_ignore_when_deleting_twice_by_id() { - ((IInitializable)Sut).Initialize(); - - var tempId = Id(); - - var assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); - await Sut.UploadAsync(tempId, 0, null, assetData); await Sut.DeleteAsync(tempId, 0, null); await Sut.DeleteAsync(tempId, 0, null); } - - protected static string Id() - { - return Guid.NewGuid().ToString(); - } } } From 77892b9caf1b2122c94525228bc1b686bf349c6f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 23 Jul 2018 18:50:26 +0200 Subject: [PATCH 07/23] Full support of AssetAlreadyExistsException --- .../Assets/AzureBlobAssetStore.cs | 71 +++++++++++-------- .../Assets/GoogleCloudAssetStore.cs | 66 +++++++++-------- .../Assets/MongoGridFsAssetStore.cs | 14 ++-- .../Assets/FolderAssetStore.cs | 8 +-- .../Assets/AssetStoreTests.cs | 5 ++ .../Assets/AzureBlobAssetStoreTests.cs | 7 +- .../Assets/FolderAssetStoreTests.cs | 8 +-- .../Assets/GoogleCloudAssetStoreTests.cs | 7 +- .../Assets/MongoGridFsAssetStoreTests.cs | 5 +- 9 files changed, 101 insertions(+), 90 deletions(-) diff --git a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs index 6daa79886..628ebdf20 100644 --- a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs +++ b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs @@ -57,44 +57,47 @@ namespace Squidex.Infrastructure.Assets return new Uri(blobContainer.StorageUri.PrimaryUri, $"/{containerName}/{blobName}").ToString(); } - public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) + public async Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { - var blobName = GetObjectName(id, version, suffix); - var blobRef = blobContainer.GetBlobReference(blobName); + var targetName = GetObjectName(id, version, suffix); + var targetBlob = blobContainer.GetBlobReference(targetName); - var tempBlob = blobContainer.GetBlockBlobReference(name); + var sourceBlob = blobContainer.GetBlockBlobReference(sourceFileName); try { - await blobRef.StartCopyAsync(tempBlob.Uri, null, null, null, null, ct); + await targetBlob.StartCopyAsync(sourceBlob.Uri, null, AccessCondition.GenerateIfNotExistsCondition(), null, null, ct); - while (blobRef.CopyState.Status == CopyStatus.Pending) + while (targetBlob.CopyState.Status == CopyStatus.Pending) { ct.ThrowIfCancellationRequested(); await Task.Delay(50); - await blobRef.FetchAttributesAsync(null, null, null, ct); + await targetBlob.FetchAttributesAsync(null, null, null, ct); } - if (blobRef.CopyState.Status != CopyStatus.Success) + if (targetBlob.CopyState.Status != CopyStatus.Success) { - throw new StorageException($"Copy of temporary file failed: {blobRef.CopyState.Status}"); + throw new StorageException($"Copy of temporary file failed: {targetBlob.CopyState.Status}"); } } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 409) + { + throw new AssetAlreadyExistsException(targetName); + } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404) { - throw new AssetNotFoundException(name, ex); + throw new AssetNotFoundException(sourceFileName, ex); } } public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { - var blobName = GetObjectName(id, version, suffix); - var blobRef = blobContainer.GetBlockBlobReference(blobName); + var blob = blobContainer.GetBlockBlobReference(GetObjectName(id, version, suffix)); try { - await blobRef.DownloadToStreamAsync(stream, null, null, null, ct); + await blob.DownloadToStreamAsync(stream, null, null, null, ct); } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404) { @@ -102,37 +105,45 @@ namespace Squidex.Infrastructure.Assets } } - public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) + public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { - var blobName = GetObjectName(id, version, suffix); - var blobRef = blobContainer.GetBlockBlobReference(blobName); - - blobRef.Metadata[AssetVersion] = version.ToString(); - blobRef.Metadata[AssetId] = id; + return UploadCoreAsync(GetObjectName(id, version, suffix), stream, ct); + } - await blobRef.UploadFromStreamAsync(stream, null, null, null, ct); - await blobRef.SetMetadataAsync(); + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) + { + return UploadCoreAsync(fileName, stream, ct); } - public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) + public Task DeleteAsync(string id, long version, string suffix) { - var tempBlob = blobContainer.GetBlockBlobReference(name); + return DeleteCoreAsync(GetObjectName(id, version, suffix)); + } - return tempBlob.UploadFromStreamAsync(stream, null, null, null, ct); + public Task DeleteAsync(string fileName) + { + return DeleteCoreAsync(fileName); } - public Task DeleteAsync(string name) + private Task DeleteCoreAsync(string blobName) { - var tempBlob = blobContainer.GetBlockBlobReference(name); + var blob = blobContainer.GetBlockBlobReference(blobName); - return tempBlob.DeleteIfExistsAsync(); + return blob.DeleteIfExistsAsync(); } - public Task DeleteAsync(string id, long version, string suffix) + private async Task UploadCoreAsync(string blobName, Stream stream, CancellationToken ct) { - var tempBlob = blobContainer.GetBlockBlobReference(GetObjectName(id, version, suffix)); + try + { + var tempBlob = blobContainer.GetBlockBlobReference(blobName); - return tempBlob.DeleteIfExistsAsync(); + await tempBlob.UploadFromStreamAsync(stream, AccessCondition.GenerateIfNotExistsCondition(), null, null, ct); + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 409) + { + throw new AssetAlreadyExistsException(blobName); + } } private string GetObjectName(string id, long version, string suffix) diff --git a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs index 7251d070a..9a8a31e7c 100644 --- a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs +++ b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs @@ -18,6 +18,8 @@ namespace Squidex.Infrastructure.Assets { public sealed class GoogleCloudAssetStore : IAssetStore, IInitializable { + private static readonly UploadObjectOptions IfNotExists = new UploadObjectOptions { IfGenerationMatch = 0 }; + private static readonly CopyObjectOptions IfNotExistsCopy = new CopyObjectOptions { IfGenerationMatch = 0 }; private readonly string bucketName; private StorageClient storageClient; @@ -49,29 +51,21 @@ namespace Squidex.Infrastructure.Assets return $"https://storage.cloud.google.com/{bucketName}/{objectName}"; } - public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) - { - return storageClient.UploadObjectAsync(bucketName, name, "application/octet-stream", stream, cancellationToken: ct); - } - - public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) - { - var objectName = GetObjectName(id, version, suffix); - - await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, cancellationToken: ct); - } - - public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) + public async Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { var objectName = GetObjectName(id, version, suffix); try { - await storageClient.CopyObjectAsync(bucketName, name, bucketName, objectName, cancellationToken: ct); + await storageClient.CopyObjectAsync(bucketName, sourceFileName, bucketName, objectName, IfNotExistsCopy, ct); } catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound) { - throw new AssetNotFoundException(name, ex); + throw new AssetNotFoundException(sourceFileName, ex); + } + catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.PreconditionFailed) + { + throw new AssetAlreadyExistsException(objectName); } } @@ -89,33 +83,47 @@ namespace Squidex.Infrastructure.Assets } } - public async Task DeleteAsync(string name) + public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) + { + return UploadCoreAsync(GetObjectName(id, version, suffix), stream, ct); + } + + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) + { + return UploadCoreAsync(fileName, stream, ct); + } + + public Task DeleteAsync(string id, long version, string suffix) + { + return DeleteCoreAsync(GetObjectName(id, version, suffix)); + } + + public Task DeleteAsync(string fileName) + { + return DeleteCoreAsync(fileName); + } + + private async Task UploadCoreAsync(string objectName, Stream stream, CancellationToken ct) { try { - await storageClient.DeleteObjectAsync(bucketName, name); + await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, IfNotExists, ct); } - catch (GoogleApiException ex) + catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.PreconditionFailed) { - if (ex.HttpStatusCode != HttpStatusCode.NotFound) - { - throw; - } + throw new AssetAlreadyExistsException(objectName); } } - public async Task DeleteAsync(string id, long version, string suffix) + private async Task DeleteCoreAsync(string objectName) { try { - await storageClient.DeleteObjectAsync(bucketName, GetObjectName(id, version, suffix)); + await storageClient.DeleteObjectAsync(bucketName, objectName); } - catch (GoogleApiException ex) + catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound) { - if (ex.HttpStatusCode != HttpStatusCode.NotFound) - { - throw; - } + return; } } diff --git a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs index dbc093b37..faadf950a 100644 --- a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs @@ -78,19 +78,14 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) - { - return UploadFileCoreAsync(fileName, stream, ct); - } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { return UploadFileCoreAsync(GetFileName(id, version, suffix), stream, ct); } - public Task DeleteAsync(string fileName) + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) { - return DeleteCoreAsync(fileName); + return UploadFileCoreAsync(fileName, stream, ct); } public Task DeleteAsync(string id, long version, string suffix) @@ -98,6 +93,11 @@ namespace Squidex.Infrastructure.Assets return DeleteCoreAsync(GetFileName(id, version, suffix)); } + public Task DeleteAsync(string fileName) + { + return DeleteCoreAsync(fileName); + } + private async Task DeleteCoreAsync(string id) { try diff --git a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs index 9e3772530..8bcda3ac6 100644 --- a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs @@ -98,14 +98,14 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) + public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { - return UploadCoreAsync(GetFile(fileName), stream, ct); + return UploadCoreAsync(GetFile(id, version, suffix), stream, ct); } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken)) { - return UploadCoreAsync(GetFile(id, version, suffix), stream, ct); + return UploadCoreAsync(GetFile(fileName), stream, ct); } public Task DeleteAsync(string id, long version, string suffix) diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index e384b4b34..4d9f8927b 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -31,6 +31,11 @@ namespace Squidex.Infrastructure.Assets get { return sut.Value; } } + protected string AssetId + { + get { return assetId; } + } + public abstract T CreateStore(); public abstract void Dispose(); diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AzureBlobAssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AzureBlobAssetStoreTests.cs index 9f1f47f96..0ce5b5820 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AzureBlobAssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AzureBlobAssetStoreTests.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System; using Xunit; #pragma warning disable xUnit1000 // Test classes must be public @@ -26,11 +25,9 @@ namespace Squidex.Infrastructure.Assets [Fact] public void Should_calculate_source_url() { - Sut.Initialize(); + var url = Sut.GenerateSourceUrl(AssetId, 1, null); - var id = Guid.NewGuid().ToString(); - - Assert.Equal($"http://127.0.0.1:10000/squidex-test-container/{id}_1", Sut.GenerateSourceUrl(id, 1, null)); + Assert.Equal($"http://127.0.0.1:10000/squidex-test-container/{AssetId}_1", url); } } } diff --git a/tests/Squidex.Infrastructure.Tests/Assets/FolderAssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/FolderAssetStoreTests.cs index 8885d1225..a1d221382 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/FolderAssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/FolderAssetStoreTests.cs @@ -39,19 +39,15 @@ namespace Squidex.Infrastructure.Assets [Fact] public void Should_create_directory_when_connecting() { - Sut.Initialize(); - Assert.True(Directory.Exists(testFolder)); } [Fact] public void Should_calculate_source_url() { - Sut.Initialize(); - - var id = Guid.NewGuid().ToString(); + var url = Sut.GenerateSourceUrl(AssetId, 1, null); - Assert.Equal(Path.Combine(testFolder, $"{id}_1"), Sut.GenerateSourceUrl(id, 1, null)); + Assert.Equal(Path.Combine(testFolder, $"{AssetId}_1"), url); } private static string CreateInvalidPath() diff --git a/tests/Squidex.Infrastructure.Tests/Assets/GoogleCloudAssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/GoogleCloudAssetStoreTests.cs index 858fb813c..d31420bcf 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/GoogleCloudAssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/GoogleCloudAssetStoreTests.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System; using Xunit; #pragma warning disable xUnit1000 // Test classes must be public @@ -26,11 +25,9 @@ namespace Squidex.Infrastructure.Assets [Fact] public void Should_calculate_source_url() { - Sut.Initialize(); + var url = Sut.GenerateSourceUrl(AssetId, 1, null); - var id = Guid.NewGuid().ToString(); - - Assert.Equal($"https://storage.cloud.google.com/squidex-test/{id}_1", Sut.GenerateSourceUrl(id, 1, null)); + Assert.Equal($"https://storage.cloud.google.com/squidex-test/{AssetId}_1", url); } } } diff --git a/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs index eb1fce5f4..eb101e3ef 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System; using MongoDB.Driver; using MongoDB.Driver.GridFS; using Xunit; @@ -43,9 +42,7 @@ namespace Squidex.Infrastructure.Assets [Fact] public void Should_not_calculate_source_url() { - Sut.Initialize(); - - Assert.Equal("UNSUPPORTED", Sut.GenerateSourceUrl(Guid.NewGuid().ToString(), 1, null)); + Assert.Equal("UNSUPPORTED", Sut.GenerateSourceUrl(AssetId, 1, null)); } } } \ No newline at end of file From ffbe583babd2242aa27bd12c13f9c55ea7a05b8f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Jul 2018 17:39:39 +0200 Subject: [PATCH 08/23] Tests fixed. --- .../AppGrainCleaner.cs | 5 ++ .../Backup/AppCleanerGrain.cs | 88 +++++++++++++------ .../ICleanableAppStorage.cs | 2 + .../Tags/GrainTagService.cs | 5 ++ .../AppGrainCleanerTests.cs | 6 ++ .../Tags/GrainTagServiceTests.cs | 6 ++ 6 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs b/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs index 1d777f3e1..b89ee049d 100644 --- a/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs +++ b/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs @@ -16,6 +16,11 @@ namespace Squidex.Domain.Apps.Entities { private readonly IGrainFactory grainFactory; + public string Name + { + get { return typeof(T).Name; } + } + public AppGrainCleaner(IGrainFactory grainFactory) { Guard.NotNull(grainFactory, nameof(grainFactory)); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs index d5724e662..ae8b49faf 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs @@ -19,6 +19,7 @@ using Squidex.Domain.Apps.Entities.Schemas; using Squidex.Domain.Apps.Entities.Schemas.State; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.States; using Squidex.Infrastructure.Tasks; @@ -32,6 +33,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IStore store; private readonly IEventStore eventStore; private readonly IEnumerable storages; + private readonly ISemanticLog log; private IPersistence persistence; private bool isCleaning; private State state = new State(); @@ -44,18 +46,21 @@ namespace Squidex.Domain.Apps.Entities.Backup public HashSet PendingApps { get; set; } = new HashSet(); } - public AppCleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages) + public AppCleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages, ISemanticLog log) { Guard.NotNull(grainFactory, nameof(grainFactory)); Guard.NotNull(store, nameof(store)); Guard.NotNull(storages, nameof(storages)); Guard.NotNull(eventStore, nameof(eventStore)); + Guard.NotNull(log, nameof(log)); this.grainFactory = grainFactory; this.store = store; this.storages = storages; + this.log = log; + this.eventStore = eventStore; } @@ -106,21 +111,42 @@ namespace Squidex.Domain.Apps.Entities.Backup { foreach (var appId in state.Apps.ToList()) { - try - { - await CleanAsync(appId); - - state.Apps.Remove(appId); - } - catch (NotSupportedException) - { - state.Apps.Remove(appId); - - state.PendingApps.Add(appId); - } - finally + using (Profiler.StartSession()) { - await persistence.WriteSnapshotAsync(state); + try + { + log.LogInformation(w => w + .WriteProperty("action", "cleanApp") + .WriteProperty("status", "started") + .WriteProperty("appId", appId.ToString())); + + await CleanAsync(appId); + + state.Apps.Remove(appId); + + log.LogInformation(w => + { + w.WriteProperty("action", "cleanApp"); + w.WriteProperty("status", "completed"); + w.WriteProperty("appId", appId.ToString()); + + Profiler.Session?.Write(w); + }); + } + catch (Exception ex) + { + state.PendingApps.Add(appId); + + log.LogError(ex, w => w + .WriteProperty("action", "cleanApp") + .WriteProperty("appId", appId.ToString())); + } + finally + { + state.Apps.Remove(appId); + + await persistence.WriteSnapshotAsync(state); + } } } } @@ -132,25 +158,37 @@ namespace Squidex.Domain.Apps.Entities.Backup private async Task CleanAsync(Guid appId) { - await eventStore.DeleteManyAsync("AppId", appId); - - var ruleIds = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); - - foreach (var ruleId in ruleIds) + using (Profiler.Trace("DeleteEvents")) { - await store.RemoveSnapshotAsync(ruleId); + await eventStore.DeleteManyAsync("AppId", appId); } - var schemaIds = await grainFactory.GetGrain(appId).GetSchemaIdsAsync(); + using (Profiler.Trace("DeleteRules")) + { + var ruleIds = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); - foreach (var schemaId in schemaIds) + foreach (var ruleId in ruleIds) + { + await store.RemoveSnapshotAsync(ruleId); + } + } + + using (Profiler.Trace("DeleteSchemas")) { - await store.RemoveSnapshotAsync(schemaId); + var schemaIds = await grainFactory.GetGrain(appId).GetSchemaIdsAsync(); + + foreach (var schemaId in schemaIds) + { + await store.RemoveSnapshotAsync(schemaId); + } } foreach (var storage in storages) { - await storage.ClearAsync(appId); + using (Profiler.Trace($"{storage.Name}.ClearAsync")) + { + await storage.ClearAsync(appId); + } } await store.RemoveSnapshotAsync(appId); diff --git a/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs b/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs index 244388906..3e0304480 100644 --- a/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs +++ b/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs @@ -12,6 +12,8 @@ namespace Squidex.Domain.Apps.Entities { public interface ICleanableAppStorage { + string Name { get; } + Task ClearAsync(Guid appId); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs index 5eafa38e2..22f6cb017 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs @@ -17,6 +17,11 @@ namespace Squidex.Domain.Apps.Entities.Tags { private readonly IGrainFactory grainFactory; + public string Name + { + get { return "Tags"; } + } + public GrainTagService(IGrainFactory grainFactory) { Guard.NotNull(grainFactory, nameof(grainFactory)); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs index db2ed08d1..56521f789 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs @@ -28,6 +28,12 @@ namespace Squidex.Domain.Apps.Entities sut = new AppGrainCleaner(grainFactory); } + [Fact] + public void Should_provide_name() + { + Assert.Equal(typeof(ICleanableAppGrain).Name, sut.Name); + } + [Fact] public async Task Should_forward_to_index() { diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs index 64cb5eebe..d9bcddf48 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs @@ -29,6 +29,12 @@ namespace Squidex.Domain.Apps.Entities.Tags sut = new GrainTagService(grainFactory); } + [Fact] + public void Should_provide_name() + { + Assert.Equal("Tags", sut.Name); + } + [Fact] public async Task Should_call_grain_when_clearing() { From feaf4788f6d26b41d3ee7781319baa02df2d421b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Jul 2018 18:29:36 +0200 Subject: [PATCH 09/23] Stream reader. --- .../Backup/BackupGrain.cs | 13 +-- .../Backup/EventStreamReader.cs | 87 +++++++++++++++++++ .../Backup/EventStreamWriter.cs | 16 ++-- .../Backup/RestoreGrain.cs | 18 ++++ .../Backup/EventStreamTests.cs | 85 ++++++++++++++++++ 5 files changed, 202 insertions(+), 17 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs create mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index 6aa82decd..5b2346bab 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -26,7 +26,7 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Backup { [Reentrant] - public sealed class BackupGrain : Grain, IBackupGrain + public sealed class BackupGrain : GrainOfGuid, IBackupGrain { private const int MaxBackups = 10; private static readonly Duration UpdateDuration = Duration.FromSeconds(1); @@ -69,16 +69,11 @@ namespace Squidex.Domain.Apps.Entities.Backup this.log = log; } - public override Task OnActivateAsync() + public override async Task OnActivateAsync(Guid key) { - return OnActivateAsync(this.GetPrimaryKey()); - } - - public async Task OnActivateAsync(Guid appId) - { - this.appId = appId; + appId = key; - persistence = store.WithSnapshots(GetType(), appId, s => state = s); + persistence = store.WithSnapshots(GetType(), key, s => state = s); await ReadAsync(); await CleanupAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs new file mode 100644 index 000000000..ba1d4bcbf --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs @@ -0,0 +1,87 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public sealed class EventStreamReader : DisposableObjectBase + { + private const int MaxItemsPerFolder = 1000; + private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); + private readonly ZipArchive archive; + + public EventStreamReader(Stream stream) + { + archive = new ZipArchive(stream, ZipArchiveMode.Read, true); + } + + protected override void DisposeObject(bool disposing) + { + if (disposing) + { + archive.Dispose(); + } + } + + public async Task ReadEventsAsync(Func eventHandler) + { + Guard.NotNull(eventHandler, nameof(eventHandler)); + + var readEvents = 0; + var readAttachments = 0; + + while (true) + { + var eventFolder = readEvents / MaxItemsPerFolder; + var eventPath = $"events/{eventFolder}/{readEvents}.json"; + var eventEntry = archive.GetEntry(eventPath); + + if (eventEntry == null) + { + break; + } + + EventData eventData; + + using (var stream = eventEntry.Open()) + { + using (var textReader = new StreamReader(stream)) + { + eventData = (EventData)JsonSerializer.Deserialize(textReader, typeof(EventData)); + } + } + + readEvents++; + + var attachmentFolder = readAttachments / MaxItemsPerFolder; + var attachmentPath = $"attachments/{attachmentFolder}/{readEvents}.blob"; + var attachmentEntry = archive.GetEntry(attachmentPath); + + if (attachmentEntry != null) + { + using (var stream = attachmentEntry.Open()) + { + await eventHandler(eventData, stream); + + readAttachments++; + } + } + else + { + await eventHandler(eventData, null); + } + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs index 7133985ac..27ea2f9f2 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs @@ -28,6 +28,14 @@ namespace Squidex.Domain.Apps.Entities.Backup archive = new ZipArchive(stream, ZipArchiveMode.Update, true); } + protected override void DisposeObject(bool disposing) + { + if (disposing) + { + archive.Dispose(); + } + } + public async Task WriteEventAsync(EventData eventData, Func attachment = null) { var eventObject = @@ -67,13 +75,5 @@ namespace Squidex.Domain.Apps.Entities.Backup writtenAttachments++; } } - - protected override void DisposeObject(bool disposing) - { - if (disposing) - { - archive.Dispose(); - } - } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs new file mode 100644 index 000000000..eb1d5757b --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -0,0 +1,18 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public sealed class RestoreGrain : GrainOfString + { + public RestoreGrain() + { + } + } +} diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs new file mode 100644 index 000000000..816daa499 --- /dev/null +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs @@ -0,0 +1,85 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Squidex.Infrastructure.EventSourcing; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public class EventStreamTests + { + public sealed class EventInfo + { + public EventData Data { get; set; } + + public byte[] Attachment { get; set; } + } + + [Fact] + public async Task Should_write_and_read_events() + { + var stream = new MemoryStream(); + + var sourceEvents = new List(); + + for (var i = 0; i < 1000; i++) + { + var eventData = new EventData { Type = i.ToString(), Metadata = i, Payload = i }; + var eventInfo = new EventInfo { Data = eventData }; + + if (i % 10 == 0) + { + eventInfo.Attachment = new byte[] { (byte)i }; + } + + sourceEvents.Add(eventInfo); + } + + using (var reader = new EventStreamWriter(stream)) + { + foreach (var @event in sourceEvents) + { + if (@event.Attachment == null) + { + await reader.WriteEventAsync(@event.Data); + } + else + { + await reader.WriteEventAsync(@event.Data, s => s.WriteAsync(@event.Attachment, 0, 1)); + } + } + } + + stream.Position = 0; + + var readEvents = new List(); + + using (var reader = new EventStreamReader(stream)) + { + await reader.ReadEventsAsync(async (eventData, attachment) => + { + var eventInfo = new EventInfo { Data = eventData }; + + if (attachment != null) + { + eventInfo.Attachment = new byte[1]; + + await attachment.ReadAsync(eventInfo.Attachment, 0, 1); + } + + readEvents.Add(eventInfo); + }); + } + + readEvents.Should().BeEquivalentTo(sourceEvents); + } + } +} From 191b29c1c737501c9d6f6674c93170c5fbbe6d4a Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Jul 2018 10:58:48 +0200 Subject: [PATCH 10/23] Extracted restore behavior. --- .../Backup/BackupGrain.cs | 25 ++- .../Backup/EventStreamReader.cs | 10 +- .../Backup/EventStreamWriter.cs | 12 +- .../Backup/Handlers/HandlerBase.cs | 55 +++++ .../Backup/Handlers/RestoreApp.cs | 50 +++++ .../Backup/Handlers/RestoreAssets.cs | 72 +++++++ .../Backup/Handlers/RestoreContents.cs | 54 +++++ .../Backup/Handlers/RestoreRules.cs | 70 ++++++ .../Backup/Handlers/RestoreSchemas.cs | 75 +++++++ .../Backup/IRestoreGrain.cs | 21 ++ .../Backup/IRestoreHandler.cs | 24 +++ .../Backup/IRestoreJob.cs | 23 ++ .../Backup/RestoreGrain.cs | 203 +++++++++++++++++- .../Backup/State/RestoreState.cs | 17 ++ .../Backup/State/RestoreStateJob.cs | 38 ++++ .../Tags/GrainTagService.cs | 5 + .../Tags/ITagGrain.cs | 1 + .../Tags/ITagService.cs | 2 + .../Tags/TagGrain.cs | 5 + .../EventSourcing/Formatter.cs | 1 + .../EventSourcing/MongoEventStore_Reader.cs | 4 +- .../EventSourcing/StoredEvent.cs | 7 +- .../Backup/EventStreamTests.cs | 12 +- .../Grains/EventConsumerGrainTests.cs | 12 +- .../EventSourcing/RetrySubscriptionTests.cs | 4 +- .../States/PersistenceEventSourcingTests.cs | 4 +- 26 files changed, 759 insertions(+), 47 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/State/RestoreState.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index 5b2346bab..12c601806 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -11,7 +11,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using NodaTime; -using Orleans; using Orleans.Concurrency; using Squidex.Domain.Apps.Entities.Backup.State; using Squidex.Domain.Apps.Events; @@ -191,28 +190,28 @@ namespace Squidex.Domain.Apps.Entities.Backup var assetVersion = 0L; var assetId = Guid.Empty; - if (parsedEvent.Payload is AssetCreated assetCreated) + switch (parsedEvent.Payload) { - assetId = assetCreated.AssetId; - assetVersion = assetCreated.FileVersion; + case AssetCreated assetCreated: + assetId = assetCreated.AssetId; + assetVersion = assetCreated.FileVersion; + break; + case AssetUpdated asetUpdated: + assetId = asetUpdated.AssetId; + assetVersion = asetUpdated.FileVersion; + break; } - if (parsedEvent.Payload is AssetUpdated asetUpdated) + await writer.WriteEventAsync(@event, attachment => { - assetId = asetUpdated.AssetId; - assetVersion = asetUpdated.FileVersion; - } - - await writer.WriteEventAsync(eventData, async attachmentStream => - { - await assetStore.DownloadAsync(assetId.ToString(), assetVersion, null, attachmentStream); + return assetStore.DownloadAsync(assetId.ToString(), assetVersion, null, attachment); }); job.HandledAssets++; } else { - await writer.WriteEventAsync(eventData); + await writer.WriteEventAsync(@event); } job.HandledEvents++; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs index ba1d4bcbf..836accdff 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs @@ -34,7 +34,7 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public async Task ReadEventsAsync(Func eventHandler) + public async Task ReadEventsAsync(Func eventHandler) { Guard.NotNull(eventHandler, nameof(eventHandler)); @@ -52,18 +52,16 @@ namespace Squidex.Domain.Apps.Entities.Backup break; } - EventData eventData; + StoredEvent eventData; using (var stream = eventEntry.Open()) { using (var textReader = new StreamReader(stream)) { - eventData = (EventData)JsonSerializer.Deserialize(textReader, typeof(EventData)); + eventData = (StoredEvent)JsonSerializer.Deserialize(textReader, typeof(StoredEvent)); } } - readEvents++; - var attachmentFolder = readAttachments / MaxItemsPerFolder; var attachmentPath = $"attachments/{attachmentFolder}/{readEvents}.blob"; var attachmentEntry = archive.GetEntry(attachmentPath); @@ -81,6 +79,8 @@ namespace Squidex.Domain.Apps.Entities.Backup { await eventHandler(eventData, null); } + + readEvents++; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs index 27ea2f9f2..2ca38b729 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs @@ -36,13 +36,9 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public async Task WriteEventAsync(EventData eventData, Func attachment = null) + public async Task WriteEventAsync(StoredEvent storedEvent, Func attachment = null) { - var eventObject = - new JObject( - new JProperty("type", eventData.Type), - new JProperty("payload", eventData.Payload), - new JProperty("metadata", eventData.Metadata)); + var eventObject = JObject.FromObject(storedEvent); var eventFolder = writtenEvents / MaxItemsPerFolder; var eventPath = $"events/{eventFolder}/{writtenEvents}.json"; @@ -59,8 +55,6 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - writtenEvents++; - if (attachment != null) { var attachmentFolder = writtenAttachments / MaxItemsPerFolder; @@ -74,6 +68,8 @@ namespace Squidex.Domain.Apps.Entities.Backup writtenAttachments++; } + + writtenEvents++; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs new file mode 100644 index 000000000..c43a91a10 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs @@ -0,0 +1,55 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Commands; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.Backup.Handlers +{ + public abstract class HandlerBase + { + private readonly IStore store; + + protected HandlerBase(IStore store) + { + Guard.NotNull(store, nameof(store)); + + this.store = store; + } + + protected async Task RebuildManyAsync(IEnumerable ids, Func action) + { + foreach (var id in ids) + { + await action(id); + } + } + + protected async Task RebuildAsync(Guid key, Func, TState, TState> func) where TState : IDomainState, new() + { + var state = new TState + { + Version = EtagVersion.Empty + }; + + var persistence = store.WithSnapshotsAndEventSourcing(typeof(TGrain), key, s => state = s, e => + { + state = func(e, state); + + state.Version++; + }); + + await persistence.ReadAsync(); + await persistence.WriteSnapshotAsync(state); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs new file mode 100644 index 000000000..77a65aabb --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs @@ -0,0 +1,50 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Events.Apps; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup.Handlers +{ + public sealed class RestoreApp : HandlerBase, IRestoreHandler + { + private NamedId appId; + + public string Name { get; } = "App"; + + public RestoreApp(IStore store) + : base(store) + { + } + + public Task HandleAsync(Envelope @event, Stream attachment) + { + if (@event.Payload is AppCreated appCreated) + { + appId = appCreated.AppId; + } + + return TaskHelper.Done; + } + + public Task ProcessAsync() + { + return TaskHelper.Done; + } + + public Task CompleteAsync() + { + return TaskHelper.Done; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs new file mode 100644 index 000000000..6b16947bc --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs @@ -0,0 +1,72 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Entities.Assets; +using Squidex.Domain.Apps.Entities.Assets.State; +using Squidex.Domain.Apps.Events.Assets; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup.Handlers +{ + public sealed class RestoreAssets : HandlerBase, IRestoreHandler + { + private readonly HashSet assetIds = new HashSet(); + private readonly IAssetStore assetStore; + + public string Name { get; } = "Assets"; + + public RestoreAssets(IStore store, IAssetStore assetStore) + : base(store) + { + Guard.NotNull(assetStore, nameof(assetStore)); + + this.assetStore = assetStore; + } + + public async Task HandleAsync(Envelope @event, Stream attachment) + { + var assetVersion = 0L; + var assetId = Guid.Empty; + + switch (@event.Payload) + { + case AssetCreated assetCreated: + assetId = assetCreated.AssetId; + assetVersion = assetCreated.FileVersion; + assetIds.Add(assetCreated.AssetId); + break; + case AssetUpdated asetUpdated: + assetId = asetUpdated.AssetId; + assetVersion = asetUpdated.FileVersion; + break; + } + + if (attachment != null) + { + await assetStore.UploadAsync(assetId.ToString(), assetVersion, null, attachment); + } + } + + public Task ProcessAsync() + { + return RebuildManyAsync(assetIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); + } + + public Task CompleteAsync() + { + return TaskHelper.Done; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs new file mode 100644 index 000000000..e5d7be6fd --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs @@ -0,0 +1,54 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.State; +using Squidex.Domain.Apps.Events.Contents; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup.Handlers +{ + public sealed class RestoreContents : HandlerBase, IRestoreHandler + { + private readonly HashSet contentIds = new HashSet(); + + public string Name { get; } = "Contents"; + + public RestoreContents(IStore store) + : base(store) + { + } + + public Task HandleAsync(Envelope @event, Stream attachment) + { + switch (@event.Payload) + { + case ContentCreated contentCreated: + contentIds.Add(contentCreated.ContentId); + break; + } + + return TaskHelper.Done; + } + + public Task ProcessAsync() + { + return RebuildManyAsync(contentIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); + } + + public Task CompleteAsync() + { + return TaskHelper.Done; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs new file mode 100644 index 000000000..44a280f78 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs @@ -0,0 +1,70 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Orleans; +using Squidex.Domain.Apps.Entities.Rules; +using Squidex.Domain.Apps.Entities.Rules.State; +using Squidex.Domain.Apps.Events.Apps; +using Squidex.Domain.Apps.Events.Rules; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup.Handlers +{ + public sealed class RestoreRules : HandlerBase, IRestoreHandler + { + private readonly HashSet ruleIds = new HashSet(); + private readonly IGrainFactory grainFactory; + private Guid appId; + + public string Name { get; } = "Rules"; + + public RestoreRules(IStore store, IGrainFactory grainFactory) + : base(store) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + + this.grainFactory = grainFactory; + } + + public Task HandleAsync(Envelope @event, Stream attachment) + { + switch (@event.Payload) + { + case AppCreated appCreated: + appId = appCreated.AppId.Id; + break; + case RuleCreated ruleCreated: + ruleIds.Add(ruleCreated.RuleId); + break; + case RuleDeleted ruleDeleted: + ruleIds.Remove(ruleDeleted.RuleId); + break; + } + + return TaskHelper.Done; + } + + public async Task ProcessAsync() + { + await RebuildManyAsync(ruleIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); + + await grainFactory.GetGrain(appId).RebuildAsync(ruleIds); + } + + public Task CompleteAsync() + { + return TaskHelper.Done; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs new file mode 100644 index 000000000..c56924e8b --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs @@ -0,0 +1,75 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Orleans; +using Squidex.Domain.Apps.Core.Schemas; +using Squidex.Domain.Apps.Entities.Schemas; +using Squidex.Domain.Apps.Entities.Schemas.State; +using Squidex.Domain.Apps.Events.Apps; +using Squidex.Domain.Apps.Events.Schemas; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup.Handlers +{ + public sealed class RestoreSchemas : HandlerBase, IRestoreHandler + { + private readonly HashSet> schemaIds = new HashSet>(); + private readonly Dictionary schemasByName = new Dictionary(); + private readonly FieldRegistry fieldRegistry; + private readonly IGrainFactory grainFactory; + private Guid appId; + + public string Name { get; } = "Schemas"; + + public RestoreSchemas(IStore store, FieldRegistry fieldRegistry, IGrainFactory grainFactory) + : base(store) + { + Guard.NotNull(fieldRegistry, nameof(fieldRegistry)); + Guard.NotNull(grainFactory, nameof(grainFactory)); + + this.fieldRegistry = fieldRegistry; + + this.grainFactory = grainFactory; + } + + public Task HandleAsync(Envelope @event, Stream attachment) + { + switch (@event.Payload) + { + case AppCreated appCreated: + appId = appCreated.AppId.Id; + break; + case SchemaCreated schemaCreated: + schemaIds.Add(schemaCreated.SchemaId); + schemasByName[schemaCreated.SchemaId.Name] = schemaCreated.SchemaId.Id; + break; + } + + return TaskHelper.Done; + } + + public async Task ProcessAsync() + { + await RebuildManyAsync(schemaIds.Select(x => x.Id), id => RebuildAsync(id, (e, s) => s.Apply(e, fieldRegistry))); + + await grainFactory.GetGrain(appId).RebuildAsync(schemasByName); + } + + public Task CompleteAsync() + { + return TaskHelper.Done; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs new file mode 100644 index 000000000..219169dd3 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs @@ -0,0 +1,21 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface IRestoreGrain + { + Task RestoreAsync(Uri url, RefToken user); + + Task> GetStateAsync(); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs new file mode 100644 index 000000000..ed943c250 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs @@ -0,0 +1,24 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.IO; +using System.Threading.Tasks; +using Squidex.Infrastructure.EventSourcing; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface IRestoreHandler + { + string Name { get; } + + Task HandleAsync(Envelope @event, Stream attachment); + + Task ProcessAsync(); + + Task CompleteAsync(); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs new file mode 100644 index 000000000..b3714d15d --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs @@ -0,0 +1,23 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using NodaTime; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface IRestoreJob + { + Uri Uri { get; } + + Instant Started { get; } + + bool IsFailed { get; } + + string Status { get; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index eb1d5757b..524405df2 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -5,14 +5,213 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using NodaTime; +using Orleans; +using Squidex.Domain.Apps.Entities.Backup.State; +using Squidex.Domain.Apps.Events; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Orleans; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Backup { - public sealed class RestoreGrain : GrainOfString + public sealed class RestoreGrain : GrainOfString, IRestoreGrain { - public RestoreGrain() + private static readonly Duration UpdateDuration = Duration.FromSeconds(1); + private readonly IClock clock; + private readonly IAssetStore assetStore; + private readonly IEventDataFormatter eventDataFormatter; + private readonly IAppCleanerGrain appCleaner; + private readonly ISemanticLog log; + private readonly IEventStore eventStore; + private readonly IBackupArchiveLocation backupArchiveLocation; + private readonly IStore store; + private readonly IEnumerable handlers; + private RestoreState state = new RestoreState(); + private IPersistence persistence; + + public RestoreGrain( + IAssetStore assetStore, + IBackupArchiveLocation backupArchiveLocation, + IClock clock, + IEventStore eventStore, + IEventDataFormatter eventDataFormatter, + IGrainFactory grainFactory, + IEnumerable handlers, + ISemanticLog log, + IStore store) + { + Guard.NotNull(assetStore, nameof(assetStore)); + Guard.NotNull(backupArchiveLocation, nameof(backupArchiveLocation)); + Guard.NotNull(clock, nameof(clock)); + Guard.NotNull(eventStore, nameof(eventStore)); + Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter)); + Guard.NotNull(grainFactory, nameof(grainFactory)); + Guard.NotNull(handlers, nameof(handlers)); + Guard.NotNull(store, nameof(store)); + Guard.NotNull(log, nameof(log)); + + this.assetStore = assetStore; + this.backupArchiveLocation = backupArchiveLocation; + this.clock = clock; + this.eventStore = eventStore; + this.eventDataFormatter = eventDataFormatter; + this.handlers = handlers; + this.store = store; + this.log = log; + } + + public override async Task OnActivateAsync(string key) + { + persistence = store.WithSnapshots(GetType(), key, s => state = s); + + await persistence.ReadAsync(); + + await CleanupAsync(); + } + + public Task RestoreAsync(Uri url, RefToken user) + { + if (state.Job != null) + { + throw new DomainException("A restore operation is already running."); + } + + state.Job = new RestoreStateJob { Started = clock.GetCurrentInstant(), Uri = url, User = user }; + + return ProcessAsync(); + } + + private async Task CleanupAsync() + { + if (state.Job != null) + { + state.Job.Status = "Failed due application restart"; + state.Job.IsFailed = true; + + if (state.Job.AppId != Guid.Empty) + { + appCleaner.EnqueueAppAsync(state.Job.AppId).Forget(); + } + + await persistence.WriteSnapshotAsync(state); + } + } + + private async Task ProcessAsync() + { + try + { + await DoAsync( + "Downloading Backup", + "Downloaded Backup", + DownloadAsync); + + await DoAsync( + "Reading Events", + "Readed Events", + ReadEventsAsync); + + foreach (var handler in handlers) + { + await DoAsync($"{handler.Name} Proessing", $"{handler.Name} Processed", handler.ProcessAsync); + } + + foreach (var handler in handlers) + { + await DoAsync($"{handler.Name} Completing", $"{handler.Name} Completed", handler.CompleteAsync); + } + + state.Job = null; + } + catch (Exception ex) + { + log.LogError(ex, w => w + .WriteProperty("action", "makeBackup") + .WriteProperty("status", "failed") + .WriteProperty("backupId", state.Job.Id.ToString())); + + state.Job.IsFailed = true; + + if (state.Job.AppId != Guid.Empty) + { + appCleaner.EnqueueAppAsync(state.Job.AppId).Forget(); + } + } + finally + { + await persistence.WriteSnapshotAsync(state); + } + } + + private async Task DownloadAsync() + { + using (var client = new HttpClient()) + { + using (var sourceStream = await client.GetStreamAsync(state.Job.Uri.ToString())) + { + using (var targetStream = await backupArchiveLocation.OpenStreamAsync(state.Job.Id)) + { + await sourceStream.CopyToAsync(targetStream); + } + } + } + } + + private async Task ReadEventsAsync() + { + using (var stream = await backupArchiveLocation.OpenStreamAsync(state.Job.Id)) + { + using (var reader = new EventStreamReader(stream)) + { + var eventIndex = 0; + + await reader.ReadEventsAsync(async (@event, attachment) => + { + var eventData = @event.Data; + + var parsedEvent = eventDataFormatter.Parse(eventData); + + if (parsedEvent.Payload is SquidexEvent squidexEvent) + { + squidexEvent.Actor = state.Job.User; + } + + foreach (var handler in handlers) + { + await handler.HandleAsync(parsedEvent, attachment); + } + + await eventStore.AppendAsync(Guid.NewGuid(), @event.StreamName, new List { @event.Data }); + + eventIndex++; + + state.Job.Status = $"Handled event {eventIndex}"; + }); + } + } + } + + private async Task DoAsync(string start, string end, Func action) + { + state.Job.Status = start; + + await action(); + + state.Job.Status = end; + } + + public Task> GetStateAsync() { + return Task.FromResult>(state.Job); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreState.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreState.cs new file mode 100644 index 000000000..d86a658fe --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreState.cs @@ -0,0 +1,17 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Newtonsoft.Json; + +namespace Squidex.Domain.Apps.Entities.Backup.State +{ + public class RestoreState + { + [JsonProperty] + public RestoreStateJob Job { get; set; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs new file mode 100644 index 000000000..9c794bae3 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs @@ -0,0 +1,38 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Newtonsoft.Json; +using NodaTime; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Backup.State +{ + public sealed class RestoreStateJob : IRestoreJob + { + [JsonProperty] + public Guid Id { get; set; } = Guid.NewGuid(); + + [JsonProperty] + public Guid AppId { get; set; } + + [JsonProperty] + public RefToken User { get; set; } + + [JsonProperty] + public Uri Uri { get; set; } + + [JsonProperty] + public Instant Started { get; set; } + + [JsonProperty] + public string Status { get; set; } + + [JsonProperty] + public bool IsFailed { get; set; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs index 22f6cb017..490189f50 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs @@ -49,6 +49,11 @@ namespace Squidex.Domain.Apps.Entities.Tags return GetGrain(appId, group).GetTagsAsync(); } + public Task RebuildTagsAsync(Guid appId, string group, Dictionary allTags) + { + return GetGrain(appId, group).RebuildTagsAsync(allTags); + } + public Task ClearAsync(Guid appId) { return GetGrain(appId, TagGroups.Assets).ClearAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs index 2947aebda..8218255c8 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs @@ -22,5 +22,6 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> GetTagsAsync(); Task ClearAsync(); + Task RebuildTagsAsync(Dictionary allTags); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs index b4972b99c..370921b42 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs @@ -20,5 +20,7 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> DenormalizeTagsAsync(Guid appId, string group, HashSet ids); Task> GetTagsAsync(Guid appId, string group); + + Task RebuildTagsAsync(Guid appId, string group, Dictionary allTags); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs index c0f9c6fac..a40110ac4 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs @@ -58,6 +58,11 @@ namespace Squidex.Domain.Apps.Entities.Tags return persistence.DeleteAsync(); } + public Task RebuildTagsAsync(Dictionary allTags) + { + throw new NotImplementedException(); + } + public async Task> NormalizeTagsAsync(HashSet names, HashSet ids) { var result = new HashSet(); diff --git a/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Formatter.cs b/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Formatter.cs index aefd883b2..c28abf358 100644 --- a/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Formatter.cs +++ b/src/Squidex.Infrastructure.GetEventStore/EventSourcing/Formatter.cs @@ -24,6 +24,7 @@ namespace Squidex.Infrastructure.EventSourcing var eventData = new EventData { Type = @event.EventType, Payload = body, Metadata = meta }; return new StoredEvent( + @event.EventStreamId, resolvedEvent.OriginalEventNumber.ToString(), resolvedEvent.Event.EventNumber, eventData); diff --git a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs index 07861115d..a9e31eb43 100644 --- a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs +++ b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs @@ -60,7 +60,7 @@ namespace Squidex.Infrastructure.EventSourcing var eventData = e.ToEventData(); var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); - result.Add(new StoredEvent(eventToken, eventStreamOffset, eventData)); + result.Add(new StoredEvent(streamName, eventToken, eventStreamOffset, eventData)); } } } @@ -111,7 +111,7 @@ namespace Squidex.Infrastructure.EventSourcing var eventData = e.ToEventData(); var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); - await callback(new StoredEvent(eventToken, eventStreamOffset, eventData)); + await callback(new StoredEvent(commit.EventStream, eventToken, eventStreamOffset, eventData)); commitOffset++; } diff --git a/src/Squidex.Infrastructure/EventSourcing/StoredEvent.cs b/src/Squidex.Infrastructure/EventSourcing/StoredEvent.cs index 3c93e21a4..97ee0f55c 100644 --- a/src/Squidex.Infrastructure/EventSourcing/StoredEvent.cs +++ b/src/Squidex.Infrastructure/EventSourcing/StoredEvent.cs @@ -9,14 +9,17 @@ namespace Squidex.Infrastructure.EventSourcing { public sealed class StoredEvent { + public string StreamName { get; } + public string EventPosition { get; } public long EventStreamNumber { get; } public EventData Data { get; } - public StoredEvent(string eventPosition, long eventStreamNumber, EventData data) + public StoredEvent(string streamName, string eventPosition, long eventStreamNumber, EventData data) { + Guard.NotNullOrEmpty(streamName, nameof(streamName)); Guard.NotNullOrEmpty(eventPosition, nameof(eventPosition)); Guard.NotNull(data, nameof(data)); @@ -24,6 +27,8 @@ namespace Squidex.Infrastructure.EventSourcing EventPosition = eventPosition; EventStreamNumber = eventStreamNumber; + + StreamName = streamName; } } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs index 816daa499..6cadd083f 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs @@ -18,7 +18,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class EventInfo { - public EventData Data { get; set; } + public StoredEvent Stored { get; set; } public byte[] Attachment { get; set; } } @@ -33,7 +33,7 @@ namespace Squidex.Domain.Apps.Entities.Backup for (var i = 0; i < 1000; i++) { var eventData = new EventData { Type = i.ToString(), Metadata = i, Payload = i }; - var eventInfo = new EventInfo { Data = eventData }; + var eventInfo = new EventInfo { Stored = new StoredEvent("S", "1", 2, eventData) }; if (i % 10 == 0) { @@ -49,11 +49,11 @@ namespace Squidex.Domain.Apps.Entities.Backup { if (@event.Attachment == null) { - await reader.WriteEventAsync(@event.Data); + await reader.WriteEventAsync(@event.Stored); } else { - await reader.WriteEventAsync(@event.Data, s => s.WriteAsync(@event.Attachment, 0, 1)); + await reader.WriteEventAsync(@event.Stored, s => s.WriteAsync(@event.Attachment, 0, 1)); } } } @@ -64,9 +64,9 @@ namespace Squidex.Domain.Apps.Entities.Backup using (var reader = new EventStreamReader(stream)) { - await reader.ReadEventsAsync(async (eventData, attachment) => + await reader.ReadEventsAsync(async (stored, attachment) => { - var eventInfo = new EventInfo { Data = eventData }; + var eventInfo = new EventInfo { Stored = stored }; if (attachment != null) { diff --git a/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs b/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs index d6556c694..da62e3e23 100644 --- a/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs +++ b/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs @@ -173,7 +173,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains [Fact] public async Task Should_invoke_and_update_position_when_event_received() { - var @event = new StoredEvent(Guid.NewGuid().ToString(), 123, eventData); + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); await sut.OnActivateAsync(consumerName); await sut.ActivateAsync(); @@ -195,7 +195,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains A.CallTo(() => formatter.Parse(eventData, true)) .Throws(new TypeNameNotFoundException()); - var @event = new StoredEvent(Guid.NewGuid().ToString(), 123, eventData); + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); await sut.OnActivateAsync(consumerName); await sut.ActivateAsync(); @@ -214,7 +214,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains [Fact] public async Task Should_not_invoke_and_update_position_when_event_is_from_another_subscription() { - var @event = new StoredEvent(Guid.NewGuid().ToString(), 123, eventData); + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); await sut.OnActivateAsync(consumerName); await sut.ActivateAsync(); @@ -302,7 +302,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains A.CallTo(() => eventConsumer.On(envelope)) .Throws(ex); - var @event = new StoredEvent(Guid.NewGuid().ToString(), 123, eventData); + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); await sut.OnActivateAsync(consumerName); await sut.ActivateAsync(); @@ -329,7 +329,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains A.CallTo(() => formatter.Parse(eventData, true)) .Throws(ex); - var @event = new StoredEvent(Guid.NewGuid().ToString(), 123, eventData); + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); await sut.OnActivateAsync(consumerName); await sut.ActivateAsync(); @@ -356,7 +356,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains A.CallTo(() => eventConsumer.On(envelope)) .Throws(exception); - var @event = new StoredEvent(Guid.NewGuid().ToString(), 123, eventData); + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); await sut.OnActivateAsync(consumerName); await sut.ActivateAsync(); diff --git a/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs b/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs index 74a7eca02..85aee50d2 100644 --- a/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs +++ b/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs @@ -90,7 +90,7 @@ namespace Squidex.Infrastructure.EventSourcing [Fact] public async Task Should_forward_event_from_inner_subscription() { - var ev = new StoredEvent("1", 2, new EventData()); + var ev = new StoredEvent("Stream", "1", 2, new EventData()); await OnEventAsync(eventSubscription, ev); await sut.StopAsync(); @@ -102,7 +102,7 @@ namespace Squidex.Infrastructure.EventSourcing [Fact] public async Task Should_not_forward_event_when_message_is_from_another_subscription() { - var ev = new StoredEvent("1", 2, new EventData()); + var ev = new StoredEvent("Stream", "1", 2, new EventData()); await OnEventAsync(A.Fake(), ev); await sut.StopAsync(); diff --git a/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs b/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs index 3b23fb421..5957ed378 100644 --- a/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs +++ b/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs @@ -56,7 +56,7 @@ namespace Squidex.Infrastructure.States [Fact] public async Task Should_ignore_old_events() { - var storedEvent = new StoredEvent("1", 0, new EventData()); + var storedEvent = new StoredEvent("1", "1", 0, new EventData()); A.CallTo(() => eventStore.QueryAsync(key, 0)) .Returns(new List { storedEvent }); @@ -252,7 +252,7 @@ namespace Squidex.Infrastructure.States foreach (var @event in events) { var eventData = new EventData(); - var eventStored = new StoredEvent(i.ToString(), i, eventData); + var eventStored = new StoredEvent(i.ToString(), i.ToString(), i, eventData); eventsStored.Add(eventStored); From 7c1c88d4a2e6f11e6dc7c5de0a9adf9ec4234445 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 Jul 2018 16:51:21 +0200 Subject: [PATCH 11/23] Single handlers to be responsible for all backup, restore and cleanup belongings. --- .../Assets/MongoAssetRepository.cs | 5 + .../MongoContentPublishedCollection.cs | 5 - .../Contents/MongoContentRepository.cs | 7 + .../History/MongoHistoryEventRepository.cs | 5 + .../AppGrainCleaner.cs | 36 ---- .../AppProvider.cs | 3 + .../Apps/AppGrain.cs | 8 +- .../Apps/BackupApps.cs | 91 +++++++++ .../Apps/Guards/GuardApp.cs | 9 +- .../AppsByNameIndexCommandMiddleware.cs | 40 +++- .../Apps/Indexes/AppsByNameIndexGrain.cs | 28 +++ .../Apps/Indexes/IAppsByNameIndex.cs | 6 +- .../Apps/Indexes/IAppsByUserIndex.cs | 2 +- .../Assets/BackupAssets.cs | 124 ++++++++++++ .../Assets/Repositories/IAssetRepository.cs | 2 + .../Backup/AppCleanerGrain.cs | 138 ++++++------- .../Backup/BackupGrain.cs | 53 ++--- .../Backup/BackupHandler.cs | 57 ++++++ ...ndlerBase.cs => BackupHandlerWithStore.cs} | 11 +- .../{EventStreamReader.cs => BackupReader.cs} | 70 ++++--- .../{EventStreamWriter.cs => BackupWriter.cs} | 59 +++--- .../CleanerStatus.cs} | 13 +- .../Backup/Handlers/RestoreApp.cs | 50 ----- .../Backup/Handlers/RestoreAssets.cs | 72 ------- .../Backup/IAppCleanerGrain.cs | 2 + .../Backup/IRestoreHandler.cs | 24 --- .../Backup/RestoreGrain.cs | 191 ++++++++++++------ .../BackupContents.cs} | 31 +-- .../Repositories/IContentRepository.cs | 2 + .../History/BackupHistory.cs | 32 +++ .../Repositories/IHistoryEventRepository.cs | 2 + .../RestoreRules.cs => Rules/BackupRules.cs} | 40 ++-- .../Rules/Indexes/IRulesByAppIndex.cs | 2 +- .../BackupSchemas.cs} | 40 ++-- .../Schemas/Indexes/ISchemasByAppIndex.cs | 2 +- .../Tags/GrainTagService.cs | 6 +- .../Tags/ITagService.cs | 2 + src/Squidex.Infrastructure/Log/Profiler.cs | 5 + src/Squidex/Config/Domain/EntitiesServices.cs | 11 +- .../AppGrainCleanerTests.cs | 46 ----- .../Apps/AppGrainTests.cs | 6 +- .../Apps/Guards/GuardAppTests.cs | 27 +-- .../AppsByNameIndexCommandMiddlewareTests.cs | 32 +++ .../Backup/EventStreamTests.cs | 64 +++--- .../Tags/GrainTagServiceTests.cs | 2 +- .../Migrations/PopulateGrainIndexes.cs | 3 - 46 files changed, 843 insertions(+), 623 deletions(-) delete mode 100644 src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs rename src/Squidex.Domain.Apps.Entities/Backup/{Handlers/HandlerBase.cs => BackupHandlerWithStore.cs} (83%) rename src/Squidex.Domain.Apps.Entities/Backup/{EventStreamReader.cs => BackupReader.cs} (50%) rename src/Squidex.Domain.Apps.Entities/Backup/{EventStreamWriter.cs => BackupWriter.cs} (53%) rename src/Squidex.Domain.Apps.Entities/{ICleanableAppStorage.cs => Backup/CleanerStatus.cs} (66%) delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs rename src/Squidex.Domain.Apps.Entities/{Backup/Handlers/RestoreContents.cs => Contents/BackupContents.cs} (60%) create mode 100644 src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs rename src/Squidex.Domain.Apps.Entities/{Backup/Handlers/RestoreRules.cs => Rules/BackupRules.cs} (65%) rename src/Squidex.Domain.Apps.Entities/{Backup/Handlers/RestoreSchemas.cs => Schemas/BackupSchemas.cs} (69%) delete mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index d3e85b6e8..b3d54fa2c 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -120,5 +120,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets return assetEntity; } } + + public Task RemoveAsync(Guid appId) + { + return Collection.DeleteManyAsync(x => x.IndexedAppId == appId); + } } } diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentPublishedCollection.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentPublishedCollection.cs index 5afce42e0..ec1d2da7c 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentPublishedCollection.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentPublishedCollection.cs @@ -56,10 +56,5 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return Collection.ReplaceOneAsync(x => x.Id == content.Id, content, new UpdateOptions { IsUpsert = true }); } - - public Task RemoveAsync(Guid id) - { - return Collection.DeleteOneAsync(x => x.Id == id); - } } } diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index fb19628b3..0e13c1345 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -115,6 +115,13 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } + public Task RemoveAsync(Guid appId) + { + return Task.WhenAll( + contentsDraft.RemoveAsync(appId), + contentsPublished.RemoveAsync(appId)); + } + public Task ClearAsync() { return Task.WhenAll( diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs index e0c172251..fb40a3062 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs @@ -112,5 +112,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.History } } } + + public Task RemoveAsync(Guid appId) + { + return Collection.DeleteManyAsync(x => x.AppId == appId); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs b/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs deleted file mode 100644 index b89ee049d..000000000 --- a/src/Squidex.Domain.Apps.Entities/AppGrainCleaner.cs +++ /dev/null @@ -1,36 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using Orleans; -using Squidex.Infrastructure; - -namespace Squidex.Domain.Apps.Entities -{ - public sealed class AppGrainCleaner : ICleanableAppStorage where T : ICleanableAppGrain - { - private readonly IGrainFactory grainFactory; - - public string Name - { - get { return typeof(T).Name; } - } - - public AppGrainCleaner(IGrainFactory grainFactory) - { - Guard.NotNull(grainFactory, nameof(grainFactory)); - - this.grainFactory = grainFactory; - } - - public Task ClearAsync(Guid appId) - { - return grainFactory.GetGrain(appId).ClearAsync(); - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/AppProvider.cs b/src/Squidex.Domain.Apps.Entities/AppProvider.cs index c8c821f49..394cb55c4 100644 --- a/src/Squidex.Domain.Apps.Entities/AppProvider.cs +++ b/src/Squidex.Domain.Apps.Entities/AppProvider.cs @@ -11,8 +11,11 @@ using System.Linq; using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Entities.Apps; +using Squidex.Domain.Apps.Entities.Apps.Indexes; using Squidex.Domain.Apps.Entities.Rules; +using Squidex.Domain.Apps.Entities.Rules.Indexes; using Squidex.Domain.Apps.Entities.Schemas; +using Squidex.Domain.Apps.Entities.Schemas.Indexes; using Squidex.Infrastructure; using Squidex.Infrastructure.Caching; using Squidex.Infrastructure.Log; diff --git a/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs b/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs index c45e1bd03..1efd1e404 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs @@ -29,7 +29,6 @@ namespace Squidex.Domain.Apps.Entities.Apps public sealed class AppGrain : SquidexDomainObjectGrain, IAppGrain { private readonly InitialPatterns initialPatterns; - private readonly IAppProvider appProvider; private readonly IAppPlansProvider appPlansProvider; private readonly IAppPlanBillingManager appPlansBillingManager; private readonly IUserResolver userResolver; @@ -38,20 +37,17 @@ namespace Squidex.Domain.Apps.Entities.Apps InitialPatterns initialPatterns, IStore store, ISemanticLog log, - IAppProvider appProvider, IAppPlansProvider appPlansProvider, IAppPlanBillingManager appPlansBillingManager, IUserResolver userResolver) : base(store, log) { Guard.NotNull(initialPatterns, nameof(initialPatterns)); - Guard.NotNull(appProvider, nameof(appProvider)); Guard.NotNull(userResolver, nameof(userResolver)); Guard.NotNull(appPlansProvider, nameof(appPlansProvider)); Guard.NotNull(appPlansBillingManager, nameof(appPlansBillingManager)); this.userResolver = userResolver; - this.appProvider = appProvider; this.appPlansProvider = appPlansProvider; this.appPlansBillingManager = appPlansBillingManager; this.initialPatterns = initialPatterns; @@ -64,9 +60,9 @@ namespace Squidex.Domain.Apps.Entities.Apps switch (command) { case CreateApp createApp: - return CreateAsync(createApp, async c => + return CreateAsync(createApp, c => { - await GuardApp.CanCreate(c, appProvider); + GuardApp.CanCreate(c); Create(c); }); diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs new file mode 100644 index 000000000..2d810f524 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -0,0 +1,91 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans; +using Squidex.Domain.Apps.Entities.Apps.Indexes; +using Squidex.Domain.Apps.Entities.Apps.State; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Events.Apps; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Orleans; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.Apps +{ + public sealed class BackupApps : BackupHandlerWithStore + { + private readonly IGrainFactory grainFactory; + private readonly HashSet users = new HashSet(); + private bool isReserved; + private AppCreated appCreated; + + public BackupApps(IStore store, IGrainFactory grainFactory) + : base(store) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + + this.grainFactory = grainFactory; + } + + public override Task RemoveAsync(Guid appId) + { + return RemoveSnapshotAsync(appId); + } + + public async override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + { + switch (@event.Payload) + { + case AppCreated appCreated: + { + this.appCreated = appCreated; + + var index = grainFactory.GetGrain(SingleGrain.Id); + + if (!(isReserved = await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name))) + { + throw new DomainException("The app id or name is not available."); + } + + break; + } + + case AppContributorAssigned contributorAssigned: + users.Add(contributorAssigned.ContributorId); + break; + + case AppContributorRemoved contributorRemoved: + users.Remove(contributorRemoved.ContributorId); + break; + } + } + + public override async Task RestoreAsync(Guid appId, BackupReader reader) + { + await grainFactory.GetGrain(SingleGrain.Id).AddAppAsync(appCreated.AppId.Id, appCreated.AppId.Name); + + foreach (var user in users) + { + await grainFactory.GetGrain(user).AddAppAsync(appCreated.AppId.Id); + } + } + + public override async Task CleanupRestoreAsync(Guid appId, Exception exception) + { + if (isReserved) + { + var index = grainFactory.GetGrain(SingleGrain.Id); + + await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name); + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Guards/GuardApp.cs b/src/Squidex.Domain.Apps.Entities/Apps/Guards/GuardApp.cs index ce81ef926..49787bba8 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Guards/GuardApp.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Guards/GuardApp.cs @@ -6,7 +6,6 @@ // ========================================================================== using System; -using System.Threading.Tasks; using Squidex.Domain.Apps.Core.Apps; using Squidex.Domain.Apps.Entities.Apps.Commands; using Squidex.Domain.Apps.Entities.Apps.Services; @@ -16,20 +15,16 @@ namespace Squidex.Domain.Apps.Entities.Apps.Guards { public static class GuardApp { - public static Task CanCreate(CreateApp command, IAppProvider appProvider) + public static void CanCreate(CreateApp command) { Guard.NotNull(command, nameof(command)); - return Validate.It(() => "Cannot create app.", async e => + Validate.It(() => "Cannot create app.", e => { if (!command.Name.IsSlug()) { e("Name must be a valid slug.", nameof(command.Name)); } - else if (await appProvider.GetAppAsync(command.Name) != null) - { - e("An app with the same name already exists.", nameof(command.Name)); - } }); } diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexCommandMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexCommandMiddleware.cs index 798fd6f1b..1722bc363 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexCommandMiddleware.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexCommandMiddleware.cs @@ -28,20 +28,44 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes public async Task HandleAsync(CommandContext context, Func next) { - if (context.IsCompleted) + var createApp = context.Command as CreateApp; + + var isReserved = false; + try { - switch (context.Command) + if (createApp != null) + { + isReserved = await index.ReserveAppAsync(createApp.AppId, createApp.Name); + + if (!isReserved) + { + var error = new ValidationError("An app with the same name already exists.", nameof(createApp.Name)); + + throw new ValidationException("Cannot create app.", error); + } + } + + await next(); + + if (context.IsCompleted) { - case CreateApp createApp: + if (createApp != null) + { await index.AddAppAsync(createApp.AppId, createApp.Name); - break; - case ArchiveApp archiveApp: + } + else if (context.Command is ArchiveApp archiveApp) + { await index.RemoveAppAsync(archiveApp.AppId); - break; + } + } + } + finally + { + if (isReserved && createApp != null) + { + await index.RemoveReservationAsync(createApp.AppId, createApp.Name); } } - - await next(); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs index db53e1dc3..bcf7d791b 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs @@ -12,12 +12,15 @@ using System.Threading.Tasks; using Squidex.Infrastructure; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Apps.Indexes { public sealed class AppsByNameIndexGrain : GrainOfString, IAppsByNameIndex { private readonly IStore store; + private readonly HashSet reservedIds = new HashSet(); + private readonly HashSet reservedNames = new HashSet(); private IPersistence persistence; private State state = new State(); @@ -51,6 +54,31 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes return persistence.WriteSnapshotAsync(state); } + public Task ReserveAppAsync(Guid appId, string name) + { + var canReserve = + !state.Apps.ContainsKey(name) && + !state.Apps.Any(x => x.Value == appId) && + !reservedIds.Contains(appId) && + !reservedNames.Contains(name); + + if (canReserve) + { + reservedIds.Add(appId); + reservedNames.Add(name); + } + + return Task.FromResult(canReserve); + } + + public Task RemoveReservationAsync(Guid appId, string name) + { + reservedIds.Remove(appId); + reservedNames.Remove(name); + + return TaskHelper.Done; + } + public Task AddAppAsync(Guid appId, string name) { state.Apps[name] = appId; diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByNameIndex.cs b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByNameIndex.cs index 9ba30cd1a..2580ab694 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByNameIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByNameIndex.cs @@ -10,16 +10,20 @@ using System.Collections.Generic; using System.Threading.Tasks; using Orleans; -namespace Squidex.Domain.Apps.Entities.Apps +namespace Squidex.Domain.Apps.Entities.Apps.Indexes { public interface IAppsByNameIndex : IGrainWithStringKey { + Task ReserveAppAsync(Guid appId, string name); + Task AddAppAsync(Guid appId, string name); Task RemoveAppAsync(Guid appId); Task RebuildAsync(Dictionary apps); + Task RemoveReservationAsync(Guid appId, string name); + Task GetAppIdAsync(string name); Task> GetAppIdsAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByUserIndex.cs b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByUserIndex.cs index 8b960b13d..e769f8803 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByUserIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByUserIndex.cs @@ -10,7 +10,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Orleans; -namespace Squidex.Domain.Apps.Entities.Apps +namespace Squidex.Domain.Apps.Entities.Apps.Indexes { public interface IAppsByUserIndex : IGrainWithStringKey { diff --git a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs new file mode 100644 index 000000000..e49efeb4d --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -0,0 +1,124 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Entities.Assets.Repositories; +using Squidex.Domain.Apps.Entities.Assets.State; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Entities.Tags; +using Squidex.Domain.Apps.Events.Assets; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Assets +{ + public sealed class BackupAssets : BackupHandlerWithStore + { + private readonly HashSet assetIds = new HashSet(); + private readonly IAssetStore assetStore; + private readonly IAssetRepository assetRepository; + private readonly ITagService tagService; + private readonly IEventDataFormatter eventDataFormatter; + + public BackupAssets(IStore store, + IEventDataFormatter eventDataFormatter, + IAssetStore assetStore, + IAssetRepository assetRepository, + ITagService tagService) + : base(store) + { + Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter)); + Guard.NotNull(assetStore, nameof(assetStore)); + Guard.NotNull(assetRepository, nameof(assetRepository)); + Guard.NotNull(tagService, nameof(tagService)); + + this.eventDataFormatter = eventDataFormatter; + this.assetStore = assetStore; + this.assetRepository = assetRepository; + this.tagService = tagService; + } + + public override async Task RemoveAsync(Guid appId) + { + await tagService.ClearAsync(appId, TagGroups.Assets); + + await assetRepository.RemoveAsync(appId); + } + + public override Task BackupEventAsync(EventData @event, Guid appId, BackupWriter writer) + { + if (@event.Type == "AssetCreatedEvent" || + @event.Type == "AssetUpdatedEvent") + { + var parsedEvent = eventDataFormatter.Parse(@event); + + switch (parsedEvent.Payload) + { + case AssetCreated assetCreated: + return WriteAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, writer); + case AssetUpdated assetUpdated: + return WriteAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, writer); + } + } + + return TaskHelper.Done; + } + + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + { + switch (@event.Payload) + { + case AssetCreated assetCreated: + assetIds.Add(assetCreated.AssetId); + + return ReadAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, reader); + case AssetUpdated assetUpdated: + return ReadAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, reader); + } + + return TaskHelper.Done; + } + + public override Task RestoreAsync(Guid appId, BackupReader reader) + { + return RebuildManyAsync(assetIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); + } + + private Task WriteAssetAsync(Guid assetId, long fileVersion, BackupWriter writer) + { + return writer.WriteAttachmentAsync(GetName(assetId, fileVersion), stream => + { + return assetStore.DownloadAsync(assetId.ToString(), fileVersion, null, stream); + }); + } + + private Task ReadAssetAsync(Guid assetId, long fileVersion, BackupReader reader) + { + return reader.ReadAttachmentAsync(GetName(assetId, fileVersion), async stream => + { + try + { + await assetStore.UploadAsync(assetId.ToString(), fileVersion, null, stream); + } + catch (AssetAlreadyExistsException) + { + return; + } + }); + } + + private static string GetName(Guid assetId, long fileVersion) + { + return $"{assetId}_{fileVersion}.asset"; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Assets/Repositories/IAssetRepository.cs b/src/Squidex.Domain.Apps.Entities/Assets/Repositories/IAssetRepository.cs index f748b4936..0410a5b4f 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/Repositories/IAssetRepository.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/Repositories/IAssetRepository.cs @@ -19,5 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Repositories Task> QueryAsync(Guid appId, HashSet ids); Task FindAssetAsync(Guid id); + + Task RemoveAsync(Guid appId); } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs index ae8b49faf..14fe28fca 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs @@ -12,11 +12,6 @@ using System.Threading.Tasks; using Orleans; using Orleans.Concurrency; using Orleans.Runtime; -using Squidex.Domain.Apps.Entities.Apps.State; -using Squidex.Domain.Apps.Entities.Rules; -using Squidex.Domain.Apps.Entities.Rules.State; -using Squidex.Domain.Apps.Entities.Schemas; -using Squidex.Domain.Apps.Entities.Schemas.State; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Log; @@ -32,7 +27,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IGrainFactory grainFactory; private readonly IStore store; private readonly IEventStore eventStore; - private readonly IEnumerable storages; + private readonly IEnumerable handlers; private readonly ISemanticLog log; private IPersistence persistence; private bool isCleaning; @@ -43,21 +38,21 @@ namespace Squidex.Domain.Apps.Entities.Backup { public HashSet Apps { get; set; } = new HashSet(); - public HashSet PendingApps { get; set; } = new HashSet(); + public HashSet FailedApps { get; set; } = new HashSet(); } - public AppCleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages, ISemanticLog log) + public AppCleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable handlers, ISemanticLog log) { Guard.NotNull(grainFactory, nameof(grainFactory)); Guard.NotNull(store, nameof(store)); - Guard.NotNull(storages, nameof(storages)); + Guard.NotNull(handlers, nameof(handlers)); Guard.NotNull(eventStore, nameof(eventStore)); Guard.NotNull(log, nameof(log)); this.grainFactory = grainFactory; this.store = store; - this.storages = storages; + this.handlers = handlers; this.log = log; @@ -99,6 +94,22 @@ namespace Squidex.Domain.Apps.Entities.Backup return TaskHelper.Done; } + public Task GetStatusAsync(Guid appId) + { + if (state.Apps.Contains(appId)) + { + return Task.FromResult(CleanerStatus.Cleaning); + } + else if (state.FailedApps.Contains(appId)) + { + return Task.FromResult(CleanerStatus.Failed); + } + else + { + return Task.FromResult(CleanerStatus.Cleaned); + } + } + private async Task CleanAsync() { if (isCleaning) @@ -111,43 +122,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { foreach (var appId in state.Apps.ToList()) { - using (Profiler.StartSession()) - { - try - { - log.LogInformation(w => w - .WriteProperty("action", "cleanApp") - .WriteProperty("status", "started") - .WriteProperty("appId", appId.ToString())); - - await CleanAsync(appId); - - state.Apps.Remove(appId); - - log.LogInformation(w => - { - w.WriteProperty("action", "cleanApp"); - w.WriteProperty("status", "completed"); - w.WriteProperty("appId", appId.ToString()); - - Profiler.Session?.Write(w); - }); - } - catch (Exception ex) - { - state.PendingApps.Add(appId); - - log.LogError(ex, w => w - .WriteProperty("action", "cleanApp") - .WriteProperty("appId", appId.ToString())); - } - finally - { - state.Apps.Remove(appId); - - await persistence.WriteSnapshotAsync(state); - } - } + await CleanupAppAsync(appId); } } finally @@ -156,47 +131,64 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - private async Task CleanAsync(Guid appId) + private async Task CleanupAppAsync(Guid appId) { - using (Profiler.Trace("DeleteEvents")) + using (Profiler.StartSession()) { - await eventStore.DeleteManyAsync("AppId", appId); - } + try + { + log.LogInformation(w => w + .WriteProperty("action", "cleanApp") + .WriteProperty("status", "started") + .WriteProperty("appId", appId.ToString())); - using (Profiler.Trace("DeleteRules")) - { - var ruleIds = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); + await CleanupCoreAsync(appId); - foreach (var ruleId in ruleIds) - { - await store.RemoveSnapshotAsync(ruleId); + log.LogInformation(w => + { + w.WriteProperty("action", "cleanApp"); + w.WriteProperty("status", "completed"); + w.WriteProperty("appId", appId.ToString()); + + Profiler.Session?.Write(w); + }); } - } + catch (Exception ex) + { + state.FailedApps.Add(appId); - using (Profiler.Trace("DeleteSchemas")) - { - var schemaIds = await grainFactory.GetGrain(appId).GetSchemaIdsAsync(); + log.LogError(ex, w => + { + w.WriteProperty("action", "cleanApp"); + w.WriteProperty("status", "failed"); + w.WriteProperty("appId", appId.ToString()); - foreach (var schemaId in schemaIds) + Profiler.Session?.Write(w); + }); + } + finally { - await store.RemoveSnapshotAsync(schemaId); + state.Apps.Remove(appId); + + await persistence.WriteSnapshotAsync(state); } } + } - foreach (var storage in storages) + private async Task CleanupCoreAsync(Guid appId) + { + using (Profiler.Trace("DeleteEvents")) { - using (Profiler.Trace($"{storage.Name}.ClearAsync")) + await eventStore.DeleteManyAsync("AppId", appId); + } + + foreach (var handler in handlers) + { + using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RemoveAsync))) { - await storage.ClearAsync(appId); + await handler.RemoveAsync(appId); } } - - await store.RemoveSnapshotAsync(appId); - } - - private async Task DeleteAsync(Guid id) - { - await store.RemoveSnapshotAsync(id); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index 12c601806..b6840a584 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -14,7 +14,6 @@ using NodaTime; using Orleans.Concurrency; using Squidex.Domain.Apps.Entities.Backup.State; using Squidex.Domain.Apps.Events; -using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; using Squidex.Infrastructure.EventSourcing; @@ -32,6 +31,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IClock clock; private readonly IAssetStore assetStore; private readonly IEventDataFormatter eventDataFormatter; + private readonly IEnumerable handlers; private readonly ISemanticLog log; private readonly IEventStore eventStore; private readonly IBackupArchiveLocation backupArchiveLocation; @@ -48,6 +48,7 @@ namespace Squidex.Domain.Apps.Entities.Backup IClock clock, IEventStore eventStore, IEventDataFormatter eventDataFormatter, + IEnumerable handlers, ISemanticLog log, IStore store) { @@ -56,6 +57,7 @@ namespace Squidex.Domain.Apps.Entities.Backup Guard.NotNull(clock, nameof(clock)); Guard.NotNull(eventStore, nameof(eventStore)); Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter)); + Guard.NotNull(handlers, nameof(handlers)); Guard.NotNull(store, nameof(store)); Guard.NotNull(log, nameof(log)); @@ -64,6 +66,7 @@ namespace Squidex.Domain.Apps.Entities.Backup this.clock = clock; this.eventStore = eventStore; this.eventDataFormatter = eventDataFormatter; + this.handlers = handlers; this.store = store; this.log = log; } @@ -176,45 +179,21 @@ namespace Squidex.Domain.Apps.Entities.Backup { using (var stream = await backupArchiveLocation.OpenStreamAsync(job.Id)) { - using (var writer = new EventStreamWriter(stream)) + using (var writer = new BackupWriter(stream)) { await eventStore.QueryAsync(async @event => { var eventData = @event.Data; - if (eventData.Type == "AssetCreatedEvent" || - eventData.Type == "AssetUpdatedEvent") - { - var parsedEvent = eventDataFormatter.Parse(eventData); - - var assetVersion = 0L; - var assetId = Guid.Empty; - - switch (parsedEvent.Payload) - { - case AssetCreated assetCreated: - assetId = assetCreated.AssetId; - assetVersion = assetCreated.FileVersion; - break; - case AssetUpdated asetUpdated: - assetId = asetUpdated.AssetId; - assetVersion = asetUpdated.FileVersion; - break; - } - - await writer.WriteEventAsync(@event, attachment => - { - return assetStore.DownloadAsync(assetId.ToString(), assetVersion, null, attachment); - }); - - job.HandledAssets++; - } - else + writer.WriteEvent(@event); + + foreach (var handler in handlers) { - await writer.WriteEventAsync(@event); + await handler.BackupEventAsync(eventData, appId, writer); } - job.HandledEvents++; + job.HandledEvents = writer.WrittenEvents; + job.HandledAssets = writer.WrittenAttachments; var now = clock.GetCurrentInstant(); @@ -225,6 +204,16 @@ namespace Squidex.Domain.Apps.Entities.Backup await WriteAsync(); } }, SquidexHeaders.AppId, appId.ToString(), null, currentTask.Token); + + foreach (var handler in handlers) + { + await handler.BackupAsync(appId, writer); + } + + foreach (var handler in handlers) + { + await handler.CompleteBackupAsync(appId, writer); + } } stream.Position = 0; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs new file mode 100644 index 000000000..b3df4d5d1 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs @@ -0,0 +1,57 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public abstract class BackupHandler + { + public virtual Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + { + return TaskHelper.Done; + } + + public virtual Task BackupEventAsync(EventData @event, Guid appId, BackupWriter writer) + { + return TaskHelper.Done; + } + + public virtual Task RestoreAsync(Guid appId, BackupReader reader) + { + return TaskHelper.Done; + } + + public virtual Task BackupAsync(Guid appId, BackupWriter writer) + { + return TaskHelper.Done; + } + + public virtual Task CleanupRestoreAsync(Guid appId, Exception exception) + { + return TaskHelper.Done; + } + + public virtual Task CompleteRestoreAsync(Guid appId, BackupReader reader) + { + return TaskHelper.Done; + } + + public virtual Task CompleteBackupAsync(Guid appId, BackupWriter writer) + { + return TaskHelper.Done; + } + + public virtual Task RemoveAsync(Guid appId) + { + return TaskHelper.Done; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs similarity index 83% rename from src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs rename to src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs index c43a91a10..67fd56e0f 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/HandlerBase.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs @@ -13,19 +13,24 @@ using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.States; -namespace Squidex.Domain.Apps.Entities.Backup.Handlers +namespace Squidex.Domain.Apps.Entities.Backup { - public abstract class HandlerBase + public abstract class BackupHandlerWithStore : BackupHandler { private readonly IStore store; - protected HandlerBase(IStore store) + protected BackupHandlerWithStore(IStore store) { Guard.NotNull(store, nameof(store)); this.store = store; } + protected Task RemoveSnapshotAsync(Guid id) + { + return store.RemoveSnapshotAsync(id); + } + protected async Task RebuildManyAsync(IEnumerable ids, Func action) { foreach (var id in ids) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs similarity index 50% rename from src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs rename to src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs index 836accdff..2de75b269 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamReader.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs @@ -15,13 +15,26 @@ using Squidex.Infrastructure.EventSourcing; namespace Squidex.Domain.Apps.Entities.Backup { - public sealed class EventStreamReader : DisposableObjectBase + public sealed class BackupReader : DisposableObjectBase { - private const int MaxItemsPerFolder = 1000; + private const int MaxEventsPerFolder = 1000; + private const int MaxAttachmentFolders = 1000; private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); private readonly ZipArchive archive; + private int readEvents; + private int readAttachments; - public EventStreamReader(Stream stream) + public int ReadEvents + { + get { return readEvents; } + } + + public int ReadAttachments + { + get { return readAttachments; } + } + + public BackupReader(Stream stream) { archive = new ZipArchive(stream, ZipArchiveMode.Read, true); } @@ -34,16 +47,35 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public async Task ReadEventsAsync(Func eventHandler) + public async Task ReadAttachmentAsync(string name, Func handler) { - Guard.NotNull(eventHandler, nameof(eventHandler)); + Guard.NotNullOrEmpty(name, nameof(name)); + Guard.NotNull(handler, nameof(handler)); - var readEvents = 0; - var readAttachments = 0; + var attachmentFolder = Math.Abs(name.GetHashCode() % MaxAttachmentFolders); + var attachmentPath = $"attachments/{attachmentFolder}/{name}"; + var attachmentEntry = archive.GetEntry(attachmentPath); + + if (attachmentEntry == null) + { + throw new FileNotFoundException("Cannot find attachment.", name); + } + + using (var stream = attachmentEntry.Open()) + { + await handler(stream); + } + + readAttachments++; + } + + public async Task ReadEventsAsync(Func handler) + { + Guard.NotNull(handler, nameof(handler)); while (true) { - var eventFolder = readEvents / MaxItemsPerFolder; + var eventFolder = readEvents / MaxEventsPerFolder; var eventPath = $"events/{eventFolder}/{readEvents}.json"; var eventEntry = archive.GetEntry(eventPath); @@ -52,33 +84,15 @@ namespace Squidex.Domain.Apps.Entities.Backup break; } - StoredEvent eventData; - using (var stream = eventEntry.Open()) { using (var textReader = new StreamReader(stream)) { - eventData = (StoredEvent)JsonSerializer.Deserialize(textReader, typeof(StoredEvent)); - } - } - - var attachmentFolder = readAttachments / MaxItemsPerFolder; - var attachmentPath = $"attachments/{attachmentFolder}/{readEvents}.blob"; - var attachmentEntry = archive.GetEntry(attachmentPath); - - if (attachmentEntry != null) - { - using (var stream = attachmentEntry.Open()) - { - await eventHandler(eventData, stream); + var storedEvent = (StoredEvent)JsonSerializer.Deserialize(textReader, typeof(StoredEvent)); - readAttachments++; + await handler(storedEvent); } } - else - { - await eventHandler(eventData, null); - } readEvents++; } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs similarity index 53% rename from src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs rename to src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs index 2ca38b729..d8951d403 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs @@ -10,20 +10,31 @@ using System.IO; using System.IO.Compression; using System.Threading.Tasks; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; namespace Squidex.Domain.Apps.Entities.Backup { - public sealed class EventStreamWriter : DisposableObjectBase + public sealed class BackupWriter : DisposableObjectBase { - private const int MaxItemsPerFolder = 1000; + private const int MaxEventsPerFolder = 1000; + private const int MaxAttachmentFolders = 1000; + private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); private readonly ZipArchive archive; private int writtenEvents; private int writtenAttachments; - public EventStreamWriter(Stream stream) + public int WrittenEvents + { + get { return writtenEvents; } + } + + public int WrittenAttachments + { + get { return writtenAttachments; } + } + + public BackupWriter(Stream stream) { archive = new ZipArchive(stream, ZipArchiveMode.Update, true); } @@ -36,11 +47,26 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public async Task WriteEventAsync(StoredEvent storedEvent, Func attachment = null) + public async Task WriteAttachmentAsync(string name, Func handler) { - var eventObject = JObject.FromObject(storedEvent); + Guard.NotNullOrEmpty(name, nameof(name)); + Guard.NotNull(handler, nameof(handler)); + + var attachmentFolder = Math.Abs(name.GetHashCode() % MaxAttachmentFolders); + var attachmentPath = $"attachments/{attachmentFolder}/{name}"; + var attachmentEntry = archive.CreateEntry(attachmentPath); + + using (var stream = attachmentEntry.Open()) + { + await handler(stream); + } - var eventFolder = writtenEvents / MaxItemsPerFolder; + writtenAttachments++; + } + + public void WriteEvent(StoredEvent storedEvent) + { + var eventFolder = writtenEvents / MaxEventsPerFolder; var eventPath = $"events/{eventFolder}/{writtenEvents}.json"; var eventEntry = archive.GetEntry(eventPath) ?? archive.CreateEntry(eventPath); @@ -48,25 +74,8 @@ namespace Squidex.Domain.Apps.Entities.Backup { using (var textWriter = new StreamWriter(stream)) { - using (var jsonWriter = new JsonTextWriter(textWriter)) - { - await eventObject.WriteToAsync(jsonWriter); - } - } - } - - if (attachment != null) - { - var attachmentFolder = writtenAttachments / MaxItemsPerFolder; - var attachmentPath = $"attachments/{attachmentFolder}/{writtenEvents}.blob"; - var attachmentEntry = archive.GetEntry(attachmentPath) ?? archive.CreateEntry(attachmentPath); - - using (var stream = attachmentEntry.Open()) - { - await attachment(stream); + JsonSerializer.Serialize(textWriter, storedEvent); } - - writtenAttachments++; } writtenEvents++; diff --git a/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs similarity index 66% rename from src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs rename to src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs index 3e0304480..0016388f2 100644 --- a/src/Squidex.Domain.Apps.Entities/ICleanableAppStorage.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs @@ -5,15 +5,12 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System; -using System.Threading.Tasks; - -namespace Squidex.Domain.Apps.Entities +namespace Squidex.Domain.Apps.Entities.Backup { - public interface ICleanableAppStorage + public enum CleanerStatus { - string Name { get; } - - Task ClearAsync(Guid appId); + Cleaned, + Cleaning, + Failed } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs deleted file mode 100644 index 77a65aabb..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreApp.cs +++ /dev/null @@ -1,50 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.IO; -using System.Threading.Tasks; -using Squidex.Domain.Apps.Events.Apps; -using Squidex.Infrastructure; -using Squidex.Infrastructure.EventSourcing; -using Squidex.Infrastructure.States; -using Squidex.Infrastructure.Tasks; - -namespace Squidex.Domain.Apps.Entities.Backup.Handlers -{ - public sealed class RestoreApp : HandlerBase, IRestoreHandler - { - private NamedId appId; - - public string Name { get; } = "App"; - - public RestoreApp(IStore store) - : base(store) - { - } - - public Task HandleAsync(Envelope @event, Stream attachment) - { - if (@event.Payload is AppCreated appCreated) - { - appId = appCreated.AppId; - } - - return TaskHelper.Done; - } - - public Task ProcessAsync() - { - return TaskHelper.Done; - } - - public Task CompleteAsync() - { - return TaskHelper.Done; - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs b/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs deleted file mode 100644 index 6b16947bc..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreAssets.cs +++ /dev/null @@ -1,72 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Squidex.Domain.Apps.Entities.Assets; -using Squidex.Domain.Apps.Entities.Assets.State; -using Squidex.Domain.Apps.Events.Assets; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Assets; -using Squidex.Infrastructure.EventSourcing; -using Squidex.Infrastructure.States; -using Squidex.Infrastructure.Tasks; - -namespace Squidex.Domain.Apps.Entities.Backup.Handlers -{ - public sealed class RestoreAssets : HandlerBase, IRestoreHandler - { - private readonly HashSet assetIds = new HashSet(); - private readonly IAssetStore assetStore; - - public string Name { get; } = "Assets"; - - public RestoreAssets(IStore store, IAssetStore assetStore) - : base(store) - { - Guard.NotNull(assetStore, nameof(assetStore)); - - this.assetStore = assetStore; - } - - public async Task HandleAsync(Envelope @event, Stream attachment) - { - var assetVersion = 0L; - var assetId = Guid.Empty; - - switch (@event.Payload) - { - case AssetCreated assetCreated: - assetId = assetCreated.AssetId; - assetVersion = assetCreated.FileVersion; - assetIds.Add(assetCreated.AssetId); - break; - case AssetUpdated asetUpdated: - assetId = asetUpdated.AssetId; - assetVersion = asetUpdated.FileVersion; - break; - } - - if (attachment != null) - { - await assetStore.UploadAsync(assetId.ToString(), assetVersion, null, attachment); - } - } - - public Task ProcessAsync() - { - return RebuildManyAsync(assetIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); - } - - public Task CompleteAsync() - { - return TaskHelper.Done; - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs index 0059a4629..9e35c7165 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs @@ -14,5 +14,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public interface IAppCleanerGrain : IBackgroundGrain { Task EnqueueAppAsync(Guid appId); + + Task GetStatusAsync(Guid appId); } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs deleted file mode 100644 index ed943c250..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreHandler.cs +++ /dev/null @@ -1,24 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.IO; -using System.Threading.Tasks; -using Squidex.Infrastructure.EventSourcing; - -namespace Squidex.Domain.Apps.Entities.Backup -{ - public interface IRestoreHandler - { - string Name { get; } - - Task HandleAsync(Envelope @event, Stream attachment); - - Task ProcessAsync(); - - Task CompleteAsync(); - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index 524405df2..2be9eef36 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -13,6 +13,7 @@ using NodaTime; using Orleans; using Squidex.Domain.Apps.Entities.Backup.State; using Squidex.Domain.Apps.Events; +using Squidex.Domain.Apps.Events.Apps; using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; using Squidex.Infrastructure.EventSourcing; @@ -29,12 +30,13 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IClock clock; private readonly IAssetStore assetStore; private readonly IEventDataFormatter eventDataFormatter; + private readonly IGrainFactory grainFactory; private readonly IAppCleanerGrain appCleaner; private readonly ISemanticLog log; private readonly IEventStore eventStore; private readonly IBackupArchiveLocation backupArchiveLocation; private readonly IStore store; - private readonly IEnumerable handlers; + private readonly IEnumerable handlers; private RestoreState state = new RestoreState(); private IPersistence persistence; @@ -45,9 +47,9 @@ namespace Squidex.Domain.Apps.Entities.Backup IEventStore eventStore, IEventDataFormatter eventDataFormatter, IGrainFactory grainFactory, - IEnumerable handlers, + IEnumerable handlers, ISemanticLog log, - IStore store) + IStore store) { Guard.NotNull(assetStore, nameof(assetStore)); Guard.NotNull(backupArchiveLocation, nameof(backupArchiveLocation)); @@ -64,9 +66,12 @@ namespace Squidex.Domain.Apps.Entities.Backup this.clock = clock; this.eventStore = eventStore; this.eventDataFormatter = eventDataFormatter; + this.grainFactory = grainFactory; this.handlers = handlers; this.store = store; this.log = log; + + appCleaner = grainFactory.GetGrain(SingleGrain.Id); } public override async Task OnActivateAsync(string key) @@ -97,10 +102,7 @@ namespace Squidex.Domain.Apps.Entities.Backup state.Job.Status = "Failed due application restart"; state.Job.IsFailed = true; - if (state.Job.AppId != Guid.Empty) - { - appCleaner.EnqueueAppAsync(state.Job.AppId).Forget(); - } + TryCleanup(); await persistence.WriteSnapshotAsync(state); } @@ -108,48 +110,96 @@ namespace Squidex.Domain.Apps.Entities.Backup private async Task ProcessAsync() { - try + using (Profiler.StartSession()) { - await DoAsync( - "Downloading Backup", - "Downloaded Backup", - DownloadAsync); + try + { + log.LogInformation(w => w + .WriteProperty("action", "restore") + .WriteProperty("status", "started") + .WriteProperty("url", state.Job.Uri.ToString())); - await DoAsync( - "Reading Events", - "Readed Events", - ReadEventsAsync); + state.Job.Status = "Downloading Backup"; - foreach (var handler in handlers) - { - await DoAsync($"{handler.Name} Proessing", $"{handler.Name} Processed", handler.ProcessAsync); - } + using (Profiler.Trace("Download")) + { + await DownloadAsync(); + } - foreach (var handler in handlers) - { - await DoAsync($"{handler.Name} Completing", $"{handler.Name} Completed", handler.CompleteAsync); + state.Job.Status = "Downloaded Backup"; + + using (var stream = await backupArchiveLocation.OpenStreamAsync(state.Job.Id)) + { + using (var reader = new BackupReader(stream)) + { + using (Profiler.Trace("ReadEvents")) + { + await ReadEventsAsync(reader); + } + + state.Job.Status = "Events read"; + + foreach (var handler in handlers) + { + using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RestoreAsync))) + { + await handler.RestoreAsync(state.Job.AppId, reader); + } + + state.Job.Status = $"{handler} Processed"; + } + + foreach (var handler in handlers) + { + using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.CompleteRestoreAsync))) + { + await handler.CompleteRestoreAsync(state.Job.AppId, reader); + } + + state.Job.Status = $"{handler} Completed"; + } + } + } + + state.Job = null; + + log.LogInformation(w => + { + w.WriteProperty("action", "restore"); + w.WriteProperty("status", "completed"); + w.WriteProperty("url", state.Job.Uri.ToString()); + + Profiler.Session?.Write(w); + }); } + catch (Exception ex) + { + state.Job.IsFailed = true; - state.Job = null; - } - catch (Exception ex) - { - log.LogError(ex, w => w - .WriteProperty("action", "makeBackup") - .WriteProperty("status", "failed") - .WriteProperty("backupId", state.Job.Id.ToString())); + if (state.Job.AppId != Guid.Empty) + { + foreach (var handler in handlers) + { + await handler.CleanupRestoreAsync(state.Job.AppId, ex); + } + } - state.Job.IsFailed = true; + TryCleanup(); + + log.LogError(ex, w => + { + w.WriteProperty("action", "retore"); + w.WriteProperty("status", "failed"); + w.WriteProperty("url", state.Job.Uri.ToString()); - if (state.Job.AppId != Guid.Empty) + Profiler.Session?.Write(w); + }); + } + finally { - appCleaner.EnqueueAppAsync(state.Job.AppId).Forget(); + await persistence.WriteSnapshotAsync(state); } } - finally - { - await persistence.WriteSnapshotAsync(state); - } } private async Task DownloadAsync() @@ -166,47 +216,58 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - private async Task ReadEventsAsync() + private async Task ReadEventsAsync(BackupReader reader) { - using (var stream = await backupArchiveLocation.OpenStreamAsync(state.Job.Id)) + await reader.ReadEventsAsync(async (storedEvent) => { - using (var reader = new EventStreamReader(stream)) + var eventData = storedEvent.Data; + var eventParsed = eventDataFormatter.Parse(eventData); + + if (eventParsed.Payload is SquidexEvent squidexEvent) + { + squidexEvent.Actor = state.Job.User; + } + else if (eventParsed.Payload is AppCreated appCreated) { - var eventIndex = 0; + state.Job.AppId = appCreated.AppId.Id; - await reader.ReadEventsAsync(async (@event, attachment) => - { - var eventData = @event.Data; + await CheckCleanupStatus(); + } - var parsedEvent = eventDataFormatter.Parse(eventData); + foreach (var handler in handlers) + { + await handler.RestoreEventAsync(eventParsed, state.Job.AppId, reader); + } - if (parsedEvent.Payload is SquidexEvent squidexEvent) - { - squidexEvent.Actor = state.Job.User; - } + await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, new List { storedEvent.Data }); - foreach (var handler in handlers) - { - await handler.HandleAsync(parsedEvent, attachment); - } + state.Job.Status = $"Handled event {reader.ReadEvents} events and {reader.ReadAttachments} attachments"; + }); + } - await eventStore.AppendAsync(Guid.NewGuid(), @event.StreamName, new List { @event.Data }); + private async Task CheckCleanupStatus() + { + var cleaner = grainFactory.GetGrain(SingleGrain.Id); - eventIndex++; + var status = await cleaner.GetStatusAsync(state.Job.AppId); - state.Job.Status = $"Handled event {eventIndex}"; - }); - } + if (status == CleanerStatus.Cleaning) + { + throw new DomainException("The app is removed in the background."); + } + + if (status == CleanerStatus.Cleaning) + { + throw new DomainException("The app could not be cleaned."); } } - private async Task DoAsync(string start, string end, Func action) + private void TryCleanup() { - state.Job.Status = start; - - await action(); - - state.Job.Status = end; + if (state.Job.AppId != Guid.Empty) + { + appCleaner.EnqueueAppAsync(state.Job.AppId).Forget(); + } } public Task> GetStateAsync() diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs similarity index 60% rename from src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs rename to src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs index e5d7be6fd..2ae90e535 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreContents.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs @@ -7,29 +7,37 @@ using System; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; -using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Entities.Contents.Repositories; using Squidex.Domain.Apps.Entities.Contents.State; using Squidex.Domain.Apps.Events.Contents; +using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.States; using Squidex.Infrastructure.Tasks; -namespace Squidex.Domain.Apps.Entities.Backup.Handlers +namespace Squidex.Domain.Apps.Entities.Contents { - public sealed class RestoreContents : HandlerBase, IRestoreHandler + public sealed class BackupContents : BackupHandlerWithStore { private readonly HashSet contentIds = new HashSet(); + private readonly IContentRepository contentRepository; - public string Name { get; } = "Contents"; - - public RestoreContents(IStore store) + public BackupContents(IStore store, IContentRepository contentRepository) : base(store) { + Guard.NotNull(contentRepository, nameof(contentRepository)); + + this.contentRepository = contentRepository; + } + + public override Task RemoveAsync(Guid appId) + { + return contentRepository.RemoveAsync(appId); } - public Task HandleAsync(Envelope @event, Stream attachment) + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) { @@ -41,14 +49,9 @@ namespace Squidex.Domain.Apps.Entities.Backup.Handlers return TaskHelper.Done; } - public Task ProcessAsync() + public override Task RestoreAsync(Guid appId, BackupReader reader) { return RebuildManyAsync(contentIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); } - - public Task CompleteAsync() - { - return TaskHelper.Done; - } } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs b/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs index 36da4d974..819161319 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs @@ -28,5 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Repositories Task FindContentAsync(IAppEntity app, ISchemaEntity schema, Status[] status, Guid id); Task QueryScheduledWithoutDataAsync(Instant now, Func callback); + + Task RemoveAsync(Guid appId); } } diff --git a/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs b/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs new file mode 100644 index 000000000..29b7bafc5 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs @@ -0,0 +1,32 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Entities.History.Repositories; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.History +{ + public sealed class BackupHistory : BackupHandler + { + private readonly IHistoryEventRepository historyEventRepository; + + public BackupHistory(IHistoryEventRepository historyEventRepository) + { + Guard.NotNull(historyEventRepository, nameof(historyEventRepository)); + + this.historyEventRepository = historyEventRepository; + } + + public override Task RemoveAsync(Guid appId) + { + return historyEventRepository.RemoveAsync(appId); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs b/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs index 2810f7e18..02a8d6e5a 100644 --- a/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs +++ b/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs @@ -14,5 +14,7 @@ namespace Squidex.Domain.Apps.Entities.History.Repositories public interface IHistoryEventRepository { Task> QueryByChannelAsync(Guid appId, string channelPrefix, int count); + + Task RemoveAsync(Guid appId); } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs similarity index 65% rename from src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs rename to src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs index 44a280f78..0ea1c005d 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreRules.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs @@ -7,29 +7,25 @@ using System; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; using Orleans; -using Squidex.Domain.Apps.Entities.Rules; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Entities.Rules.Indexes; using Squidex.Domain.Apps.Entities.Rules.State; -using Squidex.Domain.Apps.Events.Apps; using Squidex.Domain.Apps.Events.Rules; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.States; using Squidex.Infrastructure.Tasks; -namespace Squidex.Domain.Apps.Entities.Backup.Handlers +namespace Squidex.Domain.Apps.Entities.Rules { - public sealed class RestoreRules : HandlerBase, IRestoreHandler + public sealed class BackupRules : BackupHandlerWithStore { private readonly HashSet ruleIds = new HashSet(); private readonly IGrainFactory grainFactory; - private Guid appId; - public string Name { get; } = "Rules"; - - public RestoreRules(IStore store, IGrainFactory grainFactory) + public BackupRules(IStore store, IGrainFactory grainFactory) : base(store) { Guard.NotNull(grainFactory, nameof(grainFactory)); @@ -37,13 +33,24 @@ namespace Squidex.Domain.Apps.Entities.Backup.Handlers this.grainFactory = grainFactory; } - public Task HandleAsync(Envelope @event, Stream attachment) + public override async Task RemoveAsync(Guid appId) + { + var index = grainFactory.GetGrain(appId); + + var idsToRemove = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); + + foreach (var ruleId in idsToRemove) + { + await RemoveSnapshotAsync(ruleId); + } + + await index.ClearAsync(); + } + + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) { - case AppCreated appCreated: - appId = appCreated.AppId.Id; - break; case RuleCreated ruleCreated: ruleIds.Add(ruleCreated.RuleId); break; @@ -55,16 +62,11 @@ namespace Squidex.Domain.Apps.Entities.Backup.Handlers return TaskHelper.Done; } - public async Task ProcessAsync() + public async override Task RestoreAsync(Guid appId, BackupReader reader) { await RebuildManyAsync(ruleIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); await grainFactory.GetGrain(appId).RebuildAsync(ruleIds); } - - public Task CompleteAsync() - { - return TaskHelper.Done; - } } } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs index 00fc680b0..e1e7ac82c 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs @@ -9,7 +9,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -namespace Squidex.Domain.Apps.Entities.Rules +namespace Squidex.Domain.Apps.Entities.Rules.Indexes { public interface IRulesByAppIndex : ICleanableAppGrain { diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs similarity index 69% rename from src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs rename to src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs index c56924e8b..16d68c5db 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/Handlers/RestoreSchemas.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs @@ -7,33 +7,29 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Core.Schemas; -using Squidex.Domain.Apps.Entities.Schemas; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Entities.Schemas.Indexes; using Squidex.Domain.Apps.Entities.Schemas.State; -using Squidex.Domain.Apps.Events.Apps; using Squidex.Domain.Apps.Events.Schemas; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.States; using Squidex.Infrastructure.Tasks; -namespace Squidex.Domain.Apps.Entities.Backup.Handlers +namespace Squidex.Domain.Apps.Entities.Schemas { - public sealed class RestoreSchemas : HandlerBase, IRestoreHandler + public sealed class BackupSchemas : BackupHandlerWithStore { private readonly HashSet> schemaIds = new HashSet>(); private readonly Dictionary schemasByName = new Dictionary(); private readonly FieldRegistry fieldRegistry; private readonly IGrainFactory grainFactory; - private Guid appId; - public string Name { get; } = "Schemas"; - - public RestoreSchemas(IStore store, FieldRegistry fieldRegistry, IGrainFactory grainFactory) + public BackupSchemas(IStore store, FieldRegistry fieldRegistry, IGrainFactory grainFactory) : base(store) { Guard.NotNull(fieldRegistry, nameof(fieldRegistry)); @@ -44,13 +40,24 @@ namespace Squidex.Domain.Apps.Entities.Backup.Handlers this.grainFactory = grainFactory; } - public Task HandleAsync(Envelope @event, Stream attachment) + public override async Task RemoveAsync(Guid appId) + { + var index = grainFactory.GetGrain(appId); + + var idsToRemove = await index.GetSchemaIdsAsync(); + + foreach (var schemaId in idsToRemove) + { + await RemoveSnapshotAsync(schemaId); + } + + await index.ClearAsync(); + } + + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) { - case AppCreated appCreated: - appId = appCreated.AppId.Id; - break; case SchemaCreated schemaCreated: schemaIds.Add(schemaCreated.SchemaId); schemasByName[schemaCreated.SchemaId.Name] = schemaCreated.SchemaId.Id; @@ -60,16 +67,11 @@ namespace Squidex.Domain.Apps.Entities.Backup.Handlers return TaskHelper.Done; } - public async Task ProcessAsync() + public async override Task RestoreAsync(Guid appId, BackupReader reader) { await RebuildManyAsync(schemaIds.Select(x => x.Id), id => RebuildAsync(id, (e, s) => s.Apply(e, fieldRegistry))); await grainFactory.GetGrain(appId).RebuildAsync(schemasByName); } - - public Task CompleteAsync() - { - return TaskHelper.Done; - } } } diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs index 17da42a4e..701caa5ad 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs @@ -9,7 +9,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -namespace Squidex.Domain.Apps.Entities.Schemas +namespace Squidex.Domain.Apps.Entities.Schemas.Indexes { public interface ISchemasByAppIndex : ICleanableAppGrain { diff --git a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs index 490189f50..48d7e1e49 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs @@ -13,7 +13,7 @@ using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Tags { - public sealed class GrainTagService : ITagService, ICleanableAppStorage + public sealed class GrainTagService : ITagService { private readonly IGrainFactory grainFactory; @@ -54,9 +54,9 @@ namespace Squidex.Domain.Apps.Entities.Tags return GetGrain(appId, group).RebuildTagsAsync(allTags); } - public Task ClearAsync(Guid appId) + public Task ClearAsync(Guid appId, string group) { - return GetGrain(appId, TagGroups.Assets).ClearAsync(); + return GetGrain(appId, group).ClearAsync(); } private ITagGrain GetGrain(Guid appId, string group) diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs index 370921b42..19724eb1b 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs @@ -22,5 +22,7 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> GetTagsAsync(Guid appId, string group); Task RebuildTagsAsync(Guid appId, string group, Dictionary allTags); + + Task ClearAsync(Guid appId, string group); } } diff --git a/src/Squidex.Infrastructure/Log/Profiler.cs b/src/Squidex.Infrastructure/Log/Profiler.cs index 85c19bb2b..e420f2dea 100644 --- a/src/Squidex.Infrastructure/Log/Profiler.cs +++ b/src/Squidex.Infrastructure/Log/Profiler.cs @@ -34,6 +34,11 @@ namespace Squidex.Infrastructure.Log return Cleaner; } + public static IDisposable TraceMethod(Type type, [CallerMemberName] string memberName = null) + { + return Trace($"{type.Name}/{memberName}"); + } + public static IDisposable TraceMethod([CallerMemberName] string memberName = null) { return Trace($"{typeof(T).Name}/{memberName}"); diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index 5d58210c4..9f704a490 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -85,7 +85,7 @@ namespace Squidex.Config.Domain .AsSelf(); services.AddSingletonAs() - .As().As(); + .As(); services.AddSingletonAs() .As>(); @@ -93,15 +93,6 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As>(); - services.AddSingletonAs>() - .As(); - - services.AddSingletonAs>() - .As(); - - services.AddSingletonAs>() - .As(); - services.AddSingletonAs() .As(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs deleted file mode 100644 index 56521f789..000000000 --- a/tests/Squidex.Domain.Apps.Entities.Tests/AppGrainCleanerTests.cs +++ /dev/null @@ -1,46 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using FakeItEasy; -using Orleans; -using Xunit; - -namespace Squidex.Domain.Apps.Entities -{ - public class AppGrainCleanerTests - { - private readonly IGrainFactory grainFactory = A.Fake(); - private readonly ICleanableAppGrain index = A.Fake(); - private readonly Guid appId = Guid.NewGuid(); - private readonly AppGrainCleaner sut; - - public AppGrainCleanerTests() - { - A.CallTo(() => grainFactory.GetGrain(appId, null)) - .Returns(index); - - sut = new AppGrainCleaner(grainFactory); - } - - [Fact] - public void Should_provide_name() - { - Assert.Equal(typeof(ICleanableAppGrain).Name, sut.Name); - } - - [Fact] - public async Task Should_forward_to_index() - { - await sut.ClearAsync(appId); - - A.CallTo(() => index.ClearAsync()) - .MustHaveHappened(); - } - } -} diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/AppGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/AppGrainTests.cs index ae373ecd9..932e96fb9 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/AppGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/AppGrainTests.cs @@ -25,7 +25,6 @@ namespace Squidex.Domain.Apps.Entities.Apps { public class AppGrainTests : HandlerTestBase { - private readonly IAppProvider appProvider = A.Fake(); private readonly IAppPlansProvider appPlansProvider = A.Fake(); private readonly IAppPlanBillingManager appPlansBillingManager = A.Fake(); private readonly IUser user = A.Fake(); @@ -47,9 +46,6 @@ namespace Squidex.Domain.Apps.Entities.Apps public AppGrainTests() { - A.CallTo(() => appProvider.GetAppAsync(AppName)) - .Returns((IAppEntity)null); - A.CallTo(() => user.Id) .Returns(contributorId); @@ -62,7 +58,7 @@ namespace Squidex.Domain.Apps.Entities.Apps { patternId2, new AppPattern("Numbers", "[0-9]*") } }; - sut = new AppGrain(initialPatterns, Store, A.Dummy(), appProvider, appPlansProvider, appPlansBillingManager, userResolver); + sut = new AppGrain(initialPatterns, Store, A.Dummy(), appPlansProvider, appPlansBillingManager, userResolver); sut.OnActivateAsync(Id).Wait(); } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Guards/GuardAppTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Guards/GuardAppTests.cs index 588ab8762..36ec66cb9 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Guards/GuardAppTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Guards/GuardAppTests.cs @@ -5,8 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System.Threading.Tasks; using FakeItEasy; +using Orleans; using Squidex.Domain.Apps.Core.Apps; using Squidex.Domain.Apps.Entities.Apps.Commands; using Squidex.Domain.Apps.Entities.Apps.Services; @@ -19,18 +19,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.Guards { public class GuardAppTests { - private readonly IAppProvider apps = A.Fake(); private readonly IUserResolver users = A.Fake(); + private readonly IGrainFactory grainFactory = A.Fake(); private readonly IAppPlansProvider appPlans = A.Fake(); public GuardAppTests() { - A.CallTo(() => apps.GetAppAsync(A.Ignored)) - .Returns(Task.FromResult(null)); - - A.CallTo(() => apps.GetAppAsync("existing")) - .Returns(A.Dummy()); - A.CallTo(() => users.FindByIdOrEmailAsync(A.Ignored)) .Returns(A.Dummy()); @@ -42,29 +36,20 @@ namespace Squidex.Domain.Apps.Entities.Apps.Guards } [Fact] - public Task CanCreate_should_throw_exception_if_name_already_in_use() - { - var command = new CreateApp { Name = "existing" }; - - return ValidationAssert.ThrowsAsync(() => GuardApp.CanCreate(command, apps), - new ValidationError("An app with the same name already exists.", "Name")); - } - - [Fact] - public Task CanCreate_should_throw_exception_if_name_not_valid() + public void CanCreate_should_throw_exception_if_name_not_valid() { var command = new CreateApp { Name = "INVALID NAME" }; - return ValidationAssert.ThrowsAsync(() => GuardApp.CanCreate(command, apps), + ValidationAssert.Throws(() => GuardApp.CanCreate(command), new ValidationError("Name must be a valid slug.", "Name")); } [Fact] - public Task CanCreate_should_not_throw_exception_if_app_name_is_free() + public void CanCreate_should_not_throw_exception_if_app_name_is_valid() { var command = new CreateApp { Name = "new-app" }; - return GuardApp.CanCreate(command, apps); + GuardApp.CanCreate(command); } [Fact] diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs index d01b8e2c9..899245c67 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexCommandMiddlewareTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using FakeItEasy; using Orleans; using Squidex.Domain.Apps.Entities.Apps.Commands; +using Squidex.Infrastructure; using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Orleans; using Xunit; @@ -35,14 +36,45 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes [Fact] public async Task Should_add_app_to_index_on_create() { + A.CallTo(() => index.ReserveAppAsync(appId, "my-app")) + .Returns(true); + var context = new CommandContext(new CreateApp { AppId = appId, Name = "my-app" }, commandBus) .Complete(); await sut.HandleAsync(context); + A.CallTo(() => index.ReserveAppAsync(appId, "my-app")) + .MustHaveHappened(); + A.CallTo(() => index.AddAppAsync(appId, "my-app")) .MustHaveHappened(); + + A.CallTo(() => index.RemoveReservationAsync(appId, "my-app")) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_not_remove_reservation_when_not_reserved() + { + A.CallTo(() => index.ReserveAppAsync(appId, "my-app")) + .Returns(false); + + var context = + new CommandContext(new CreateApp { AppId = appId, Name = "my-app" }, commandBus) + .Complete(); + + await Assert.ThrowsAsync(() => sut.HandleAsync(context)); + + A.CallTo(() => index.ReserveAppAsync(appId, "my-app")) + .MustHaveHappened(); + + A.CallTo(() => index.AddAppAsync(appId, "my-app")) + .MustNotHaveHappened(); + + A.CallTo(() => index.RemoveReservationAsync(appId, "my-app")) + .MustNotHaveHappened(); } [Fact] diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs index 6cadd083f..0085d1852 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs @@ -10,72 +10,64 @@ using System.IO; using System.Threading.Tasks; using FluentAssertions; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Tasks; using Xunit; namespace Squidex.Domain.Apps.Entities.Backup { public class EventStreamTests { - public sealed class EventInfo - { - public StoredEvent Stored { get; set; } - - public byte[] Attachment { get; set; } - } - [Fact] public async Task Should_write_and_read_events() { var stream = new MemoryStream(); - var sourceEvents = new List(); + var sourceEvents = new List(); - for (var i = 0; i < 1000; i++) + using (var writer = new BackupWriter(stream)) { - var eventData = new EventData { Type = i.ToString(), Metadata = i, Payload = i }; - var eventInfo = new EventInfo { Stored = new StoredEvent("S", "1", 2, eventData) }; - - if (i % 10 == 0) + for (var i = 0; i < 1000; i++) { - eventInfo.Attachment = new byte[] { (byte)i }; - } - - sourceEvents.Add(eventInfo); - } + var eventData = new EventData { Type = i.ToString(), Metadata = i, Payload = i }; + var eventStored = new StoredEvent("S", "1", 2, eventData); - using (var reader = new EventStreamWriter(stream)) - { - foreach (var @event in sourceEvents) - { - if (@event.Attachment == null) - { - await reader.WriteEventAsync(@event.Stored); - } - else + if (i % 10 == 0) { - await reader.WriteEventAsync(@event.Stored, s => s.WriteAsync(@event.Attachment, 0, 1)); + await writer.WriteAttachmentAsync(eventData.Type, innerStream => + { + return innerStream.WriteAsync(new byte[] { (byte)i }, 0, 1); + }); } + + writer.WriteEvent(eventStored); + + sourceEvents.Add(eventStored); } } stream.Position = 0; - var readEvents = new List(); + var readEvents = new List(); - using (var reader = new EventStreamReader(stream)) + using (var reader = new BackupReader(stream)) { - await reader.ReadEventsAsync(async (stored, attachment) => + await reader.ReadEventsAsync(async @event => { - var eventInfo = new EventInfo { Stored = stored }; + var i = int.Parse(@event.Data.Type); - if (attachment != null) + if (i % 10 == 0) { - eventInfo.Attachment = new byte[1]; + await reader.ReadAttachmentAsync(@event.Data.Type, innerStream => + { + var b = innerStream.ReadByte(); + + Assert.Equal((byte)i, b); - await attachment.ReadAsync(eventInfo.Attachment, 0, 1); + return TaskHelper.Done; + }); } - readEvents.Add(eventInfo); + readEvents.Add(@event); }); } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs index d9bcddf48..563429da0 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs @@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Tags [Fact] public async Task Should_call_grain_when_clearing() { - await sut.ClearAsync(appId); + await sut.ClearAsync(appId, TagGroups.Assets); A.CallTo(() => grain.ClearAsync()) .MustHaveHappened(); diff --git a/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs b/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs index 40828a5fa..225e67a10 100644 --- a/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs +++ b/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs @@ -9,11 +9,8 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using Orleans; -using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Apps.State; -using Squidex.Domain.Apps.Entities.Rules; using Squidex.Domain.Apps.Entities.Rules.State; -using Squidex.Domain.Apps.Entities.Schemas; using Squidex.Domain.Apps.Entities.Schemas.State; using Squidex.Infrastructure; using Squidex.Infrastructure.Migrations; From 5b8fd0ce24af58b925e0b4e8f0725d6505cab996 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 30 Jul 2018 12:24:22 +0200 Subject: [PATCH 12/23] A lot of stuff --- .../Assets/BackupAssets.cs | 40 ++++++++++++- .../Backup/IBackupGrain.cs | 5 +- .../Rules/Indexes/IRulesByAppIndex.cs | 5 +- .../Schemas/Indexes/ISchemasByAppIndex.cs | 5 +- .../Tags/GrainTagService.cs | 9 ++- .../Tags/ITagGrain.cs | 5 +- .../Tags/ITagService.cs | 4 +- src/Squidex.Domain.Apps.Entities/Tags/Tag.cs | 16 +++++ .../Tags/TagGrain.cs | 22 +++---- .../Tags/TagSet.cs | 15 +++++ src/Squidex/Config/Domain/EntitiesServices.cs | 21 +++++-- .../Apps/Indexes/AppsByNameIndexGrainTests.cs | 58 +++++++++++++++++++ .../Tags/GrainTagServiceTests.cs | 20 +++++++ .../Tags/TagGrainTests.cs | 24 ++++++++ tools/Migrate_01/Migrations/AddPatterns.cs | 1 + .../Migrations/PopulateGrainIndexes.cs | 3 + 16 files changed, 228 insertions(+), 25 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Tags/Tag.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Tags/TagSet.cs diff --git a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index e49efeb4d..e9157981f 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -7,7 +7,9 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading.Tasks; +using Newtonsoft.Json; using Squidex.Domain.Apps.Entities.Assets.Repositories; using Squidex.Domain.Apps.Entities.Assets.State; using Squidex.Domain.Apps.Entities.Backup; @@ -23,6 +25,7 @@ namespace Squidex.Domain.Apps.Entities.Assets { public sealed class BackupAssets : BackupHandlerWithStore { + private static readonly JsonSerializer Serializer = JsonSerializer.Create(); private readonly HashSet assetIds = new HashSet(); private readonly IAssetStore assetStore; private readonly IAssetRepository assetRepository; @@ -73,6 +76,11 @@ namespace Squidex.Domain.Apps.Entities.Assets return TaskHelper.Done; } + public override Task BackupAsync(Guid appId, BackupWriter writer) + { + return BackupTagsAsync(appId, writer); + } + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) @@ -88,9 +96,37 @@ namespace Squidex.Domain.Apps.Entities.Assets return TaskHelper.Done; } - public override Task RestoreAsync(Guid appId, BackupReader reader) + public override async Task RestoreAsync(Guid appId, BackupReader reader) + { + await RestoreTagsAsync(appId, reader); + + await RebuildManyAsync(assetIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); + } + + private Task RestoreTagsAsync(Guid appId, BackupReader reader) + { + return reader.ReadAttachmentAsync("AssetTags.json", async stream => + { + using (var textReader = new StreamReader(stream)) + { + var tags = (TagSet)Serializer.Deserialize(textReader, typeof(TagSet)); + + await tagService.RebuildTagsAsync(appId, TagGroups.Assets, tags); + } + }); + } + + private Task BackupTagsAsync(Guid appId, BackupWriter writer) { - return RebuildManyAsync(assetIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); + return writer.WriteAttachmentAsync("AssetTags.json", async stream => + { + var tags = await tagService.GetExportableTagsAsync(appId, TagGroups.Assets); + + using (var textWriter = new StreamWriter(stream)) + { + Serializer.Serialize(textWriter, tags); + } + }); } private Task WriteAssetAsync(Guid assetId, long fileVersion, BackupWriter writer) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs index ec545f3dd..aa48fdbd4 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs @@ -8,16 +8,19 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Orleans; using Squidex.Infrastructure.Orleans; namespace Squidex.Domain.Apps.Entities.Backup { - public interface IBackupGrain : ICleanableAppGrain + public interface IBackupGrain : IGrainWithGuidKey { Task RunAsync(); Task DeleteAsync(Guid id); + Task ClearAsync(); + Task>> GetStateAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs index e1e7ac82c..a58689e4c 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs @@ -8,10 +8,11 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Orleans; namespace Squidex.Domain.Apps.Entities.Rules.Indexes { - public interface IRulesByAppIndex : ICleanableAppGrain + public interface IRulesByAppIndex : IGrainWithGuidKey { Task AddRuleAsync(Guid ruleId); @@ -19,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes Task RebuildAsync(HashSet rules); + Task ClearAsync(); + Task> GetRuleIdsAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs index 701caa5ad..0cffd11a9 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs @@ -8,10 +8,11 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Orleans; namespace Squidex.Domain.Apps.Entities.Schemas.Indexes { - public interface ISchemasByAppIndex : ICleanableAppGrain + public interface ISchemasByAppIndex : IGrainWithGuidKey { Task AddSchemaAsync(Guid schemaId, string name); @@ -19,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes Task RebuildAsync(Dictionary schemas); + Task ClearAsync(); + Task GetSchemaIdAsync(string name); Task> GetSchemaIdsAsync(); diff --git a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs index 48d7e1e49..d88dd023a 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs @@ -49,9 +49,14 @@ namespace Squidex.Domain.Apps.Entities.Tags return GetGrain(appId, group).GetTagsAsync(); } - public Task RebuildTagsAsync(Guid appId, string group, Dictionary allTags) + public Task GetExportableTagsAsync(Guid appId, string group) { - return GetGrain(appId, group).RebuildTagsAsync(allTags); + return GetGrain(appId, group).GetExportableTagsAsync(); + } + + public Task RebuildTagsAsync(Guid appId, string group, TagSet tags) + { + return GetGrain(appId, group).RebuildAsync(tags); } public Task ClearAsync(Guid appId, string group) diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs index 8218255c8..952702b10 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs @@ -21,7 +21,10 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> GetTagsAsync(); + Task GetExportableTagsAsync(); + Task ClearAsync(); - Task RebuildTagsAsync(Dictionary allTags); + + Task RebuildAsync(TagSet tags); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs index 19724eb1b..01fb3c9e8 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs @@ -21,7 +21,9 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> GetTagsAsync(Guid appId, string group); - Task RebuildTagsAsync(Guid appId, string group, Dictionary allTags); + Task GetExportableTagsAsync(Guid appId, string group); + + Task RebuildTagsAsync(Guid appId, string group, TagSet tags); Task ClearAsync(Guid appId, string group); } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/Tag.cs b/src/Squidex.Domain.Apps.Entities/Tags/Tag.cs new file mode 100644 index 000000000..60fab8187 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Tags/Tag.cs @@ -0,0 +1,16 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Domain.Apps.Entities.Tags +{ + public sealed class Tag + { + public string Name { get; set; } + + public int Count { get; set; } = 1; + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs index a40110ac4..f4d837206 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs @@ -24,14 +24,7 @@ namespace Squidex.Domain.Apps.Entities.Tags [CollectionName("Index_Tags")] public sealed class State { - public Dictionary Tags { get; set; } = new Dictionary(); - } - - public sealed class TagInfo - { - public string Name { get; set; } - - public int Count { get; set; } = 1; + public TagSet Tags { get; set; } = new TagSet(); } public TagGrain(IStore store) @@ -58,9 +51,11 @@ namespace Squidex.Domain.Apps.Entities.Tags return persistence.DeleteAsync(); } - public Task RebuildTagsAsync(Dictionary allTags) + public Task RebuildAsync(TagSet tags) { - throw new NotImplementedException(); + state.Tags = tags; + + return persistence.DeleteAsync(); } public async Task> NormalizeTagsAsync(HashSet names, HashSet ids) @@ -91,7 +86,7 @@ namespace Squidex.Domain.Apps.Entities.Tags { tagId = Guid.NewGuid().ToString(); - state.Tags.Add(tagId, new TagInfo { Name = tagName }); + state.Tags.Add(tagId, new Tag { Name = tagName }); } result.Add(tagId); @@ -159,5 +154,10 @@ namespace Squidex.Domain.Apps.Entities.Tags { return Task.FromResult(state.Tags.Values.ToDictionary(x => x.Name, x => x.Count)); } + + public Task GetExportableTagsAsync() + { + return Task.FromResult(state.Tags); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagSet.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagSet.cs new file mode 100644 index 000000000..48815870e --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagSet.cs @@ -0,0 +1,15 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; + +namespace Squidex.Domain.Apps.Entities.Tags +{ + public sealed class TagSet : Dictionary + { + } +} diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index 9f704a490..b690a4d1b 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -96,7 +96,8 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); - AddCommandPipeline(services); + services.AddCommandPipeline(); + services.AddBackupHandlers(); services.AddSingleton>(DomainObjectGrainFormatter.Format); @@ -119,7 +120,7 @@ namespace Squidex.Config.Domain }); } - private static void AddCommandPipeline(IServiceCollection services) + private static void AddCommandPipeline(this IServiceCollection services) { services.AddSingletonAs() .As(); @@ -145,6 +146,9 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); + services.AddSingletonAs() + .As(); + services.AddSingletonAs>() .As(); @@ -157,9 +161,6 @@ namespace Squidex.Config.Domain services.AddSingletonAs>() .As(); - services.AddSingletonAs() - .As(); - services.AddSingletonAs() .As(); @@ -182,6 +183,16 @@ namespace Squidex.Config.Domain .As(); } + private static void AddBackupHandlers(this IServiceCollection services) + { + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + } + public static void AddMyMigrationServices(this IServiceCollection services) { services.AddSingletonAs() diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs index 7667f15a8..368b9c417 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Indexes/AppsByNameIndexGrainTests.cs @@ -47,6 +47,64 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes .MustHaveHappened(); } + [Fact] + public async Task Should_not_be_able_to_reserve_index_if_name_taken() + { + await sut.AddAppAsync(appId2, appName1); + + Assert.False(await sut.ReserveAppAsync(appId1, appName1)); + } + + [Fact] + public async Task Should_not_be_able_to_reserve_if_name_reserved() + { + await sut.ReserveAppAsync(appId2, appName1); + + Assert.False(await sut.ReserveAppAsync(appId1, appName1)); + } + + [Fact] + public async Task Should_not_be_able_to_reserve_if_id_taken() + { + await sut.AddAppAsync(appId1, appName1); + + Assert.False(await sut.ReserveAppAsync(appId1, appName2)); + } + + [Fact] + public async Task Should_not_be_able_to_reserve_if_id_reserved() + { + await sut.ReserveAppAsync(appId1, appName1); + + Assert.False(await sut.ReserveAppAsync(appId1, appName2)); + } + + [Fact] + public async Task Should_be_able_to_reserve_if_id_and_name_not_reserved() + { + await sut.ReserveAppAsync(appId1, appName1); + + Assert.True(await sut.ReserveAppAsync(appId2, appName2)); + } + + [Fact] + public async Task Should_be_able_to_reserve_after_app_removed() + { + await sut.AddAppAsync(appId1, appName1); + await sut.RemoveAppAsync(appId1); + + Assert.True(await sut.ReserveAppAsync(appId1, appName1)); + } + + [Fact] + public async Task Should_be_able_to_reserve_after_reservation_removed() + { + await sut.ReserveAppAsync(appId1, appName1); + await sut.RemoveReservationAsync(appId1, appName1); + + Assert.True(await sut.ReserveAppAsync(appId1, appName1)); + } + [Fact] public async Task Should_remove_app_id_from_index() { diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs index 563429da0..c3667ee77 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs @@ -44,6 +44,26 @@ namespace Squidex.Domain.Apps.Entities.Tags .MustHaveHappened(); } + [Fact] + public async Task Should_call_grain_when_rebuilding() + { + var tags = new TagSet(); + + await sut.RebuildTagsAsync(appId, TagGroups.Assets, tags); + + A.CallTo(() => grain.RebuildAsync(tags)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_call_grain_when_retrieving_raw_tags() + { + await sut.GetExportableTagsAsync(appId, TagGroups.Assets); + + A.CallTo(() => grain.GetTagsAsync()) + .MustHaveHappened(); + } + [Fact] public async Task Should_call_grain_when_retrieving_tags() { diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs index 998252ada..90b9fa0c8 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs @@ -45,6 +45,30 @@ namespace Squidex.Domain.Apps.Entities.Tags .MustHaveHappened(); } + [Fact] + public async Task Should_rebuild_tags() + { + var tags = new TagSet + { + ["1"] = new Tag { Name = "tag1", Count = 1 }, + ["2"] = new Tag { Name = "tag2", Count = 2 }, + ["3"] = new Tag { Name = "tag3", Count = 6 } + }; + + await sut.RebuildAsync(tags); + + var allTags = await sut.GetTagsAsync(); + + Assert.Equal(new Dictionary + { + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 6 + }, allTags); + + Assert.Same(tags, await sut.GetExportableTagsAsync()); + } + [Fact] public async Task Should_add_tags_to_grain() { diff --git a/tools/Migrate_01/Migrations/AddPatterns.cs b/tools/Migrate_01/Migrations/AddPatterns.cs index 693435f22..4426982c6 100644 --- a/tools/Migrate_01/Migrations/AddPatterns.cs +++ b/tools/Migrate_01/Migrations/AddPatterns.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Apps.Commands; +using Squidex.Domain.Apps.Entities.Apps.Indexes; using Squidex.Infrastructure.Migrations; using Squidex.Infrastructure.Orleans; diff --git a/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs b/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs index 225e67a10..24ed5e092 100644 --- a/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs +++ b/tools/Migrate_01/Migrations/PopulateGrainIndexes.cs @@ -9,8 +9,11 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using Orleans; +using Squidex.Domain.Apps.Entities.Apps.Indexes; using Squidex.Domain.Apps.Entities.Apps.State; +using Squidex.Domain.Apps.Entities.Rules.Indexes; using Squidex.Domain.Apps.Entities.Rules.State; +using Squidex.Domain.Apps.Entities.Schemas.Indexes; using Squidex.Domain.Apps.Entities.Schemas.State; using Squidex.Infrastructure; using Squidex.Infrastructure.Migrations; From 7f4115873bb5d295b46cf66171445b2aa447152f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 30 Jul 2018 18:30:44 +0200 Subject: [PATCH 13/23] A lot of refactorings, does not build. --- .../Apps/BackupApps.cs | 4 +- .../Backup/AppCleanerGrain.cs | 53 +++-- .../Backup/BackupGrain.cs | 39 ++-- .../Backup/BackupHandler.cs | 2 + .../Backup/BackupReader.cs | 11 +- .../Backup/BackupRestoreException.cs | 31 +++ .../Backup/BackupWriter.cs | 11 +- .../Backup/Helpers/ArchiveHelper.cs | 50 +++++ .../Backup/Helpers/Downloader.cs | 66 +++++++ .../Backup/IBackupJob.cs | 2 +- .../Backup/IRestoreGrain.cs | 8 +- .../Backup/IRestoreJob.cs | 7 +- .../Backup/JobStatus.cs | 17 ++ .../Backup/RestoreGrain.cs | 185 +++++++++++------- .../Backup/State/BackupStateJob.cs | 2 +- .../Backup/State/RestoreStateJob.cs | 14 +- .../Backup/TempFolderBackupArchiveLocation.cs | 4 +- .../Tags/TagGrain.cs | 2 +- .../Tasks/TaskExtensions.cs | 16 ++ .../Backups/Models/RestoreJobDto.cs | 45 +++++ .../Backups/Models/RestoreRequest.cs | 21 ++ .../Controllers/Backups/RestoreController.cs | 86 ++++++++ src/Squidex/Config/Domain/EntitiesServices.cs | 29 ++- .../apps/pages/apps-page.component.html | 158 +++++++++------ .../apps/pages/apps-page.component.scss | 46 ++++- .../apps/pages/apps-page.component.ts | 47 ++++- .../pages/backups/backups-page.component.html | 2 +- .../pages/backups/backups-page.component.scss | 8 +- .../pages/languages/language.component.html | 2 +- .../components/history-list.component.scss | 4 + src/Squidex/app/shared/internal.ts | 1 + .../app/shared/services/backups.service.ts | 52 ++++- src/Squidex/app/shared/state/backups.forms.ts | 26 +++ 33 files changed, 832 insertions(+), 219 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/BackupRestoreException.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Helpers/ArchiveHelper.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Helpers/Downloader.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/JobStatus.cs create mode 100644 src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs create mode 100644 src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs create mode 100644 src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs create mode 100644 src/Squidex/app/shared/state/backups.forms.ts diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs index 2d810f524..3c3cfc36f 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -52,7 +52,7 @@ namespace Squidex.Domain.Apps.Entities.Apps if (!(isReserved = await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name))) { - throw new DomainException("The app id or name is not available."); + throw new BackupRestoreException("The app id or name is not available."); } break; @@ -68,7 +68,7 @@ namespace Squidex.Domain.Apps.Entities.Apps } } - public override async Task RestoreAsync(Guid appId, BackupReader reader) + public override async Task CompleteRestoreAsync(Guid appId, BackupReader reader) { await grainFactory.GetGrain(SingleGrain.Id).AddAppAsync(appCreated.AppId.Id, appCreated.AppId.Name); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs index 14fe28fca..c833350c0 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs @@ -33,7 +33,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private bool isCleaning; private State state = new State(); - [CollectionName("Index_AppsByName")] + [CollectionName("AppCleaner")] public sealed class State { public HashSet Apps { get; set; } = new HashSet(); @@ -61,35 +61,40 @@ namespace Squidex.Domain.Apps.Entities.Backup public async override Task OnActivateAsync(string key) { - await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(2)); + await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(1)); persistence = store.WithSnapshots(Guid.Empty, s => { state = s; }); - await persistence.ReadAsync(); + await ReadAsync(); - await CleanAsync(); + Clean(); } public Task EnqueueAppAsync(Guid appId) { - state.Apps.Add(appId); + if (appId != Guid.Empty) + { + state.Apps.Add(appId); + + Clean(); + } - return persistence.WriteSnapshotAsync(state); + return WriteAsync(); } public Task ActivateAsync() { - CleanAsync().Forget(); + Clean(); return TaskHelper.Done; } public Task ReceiveReminder(string reminderName, TickStatus status) { - CleanAsync().Forget(); + Clean(); return TaskHelper.Done; } @@ -100,14 +105,18 @@ namespace Squidex.Domain.Apps.Entities.Backup { return Task.FromResult(CleanerStatus.Cleaning); } - else if (state.FailedApps.Contains(appId)) + + if (state.FailedApps.Contains(appId)) { return Task.FromResult(CleanerStatus.Failed); } - else - { - return Task.FromResult(CleanerStatus.Cleaned); - } + + return Task.FromResult(CleanerStatus.Cleaned); + } + + private void Clean() + { + CleanAsync().Forget(); } private async Task CleanAsync() @@ -142,7 +151,7 @@ namespace Squidex.Domain.Apps.Entities.Backup .WriteProperty("status", "started") .WriteProperty("appId", appId.ToString())); - await CleanupCoreAsync(appId); + await CleanupAppCoreAsync(appId); log.LogInformation(w => { @@ -170,16 +179,16 @@ namespace Squidex.Domain.Apps.Entities.Backup { state.Apps.Remove(appId); - await persistence.WriteSnapshotAsync(state); + await WriteAsync(); } } } - private async Task CleanupCoreAsync(Guid appId) + private async Task CleanupAppCoreAsync(Guid appId) { using (Profiler.Trace("DeleteEvents")) { - await eventStore.DeleteManyAsync("AppId", appId); + await eventStore.DeleteManyAsync("AppId", appId.ToString()); } foreach (var handler in handlers) @@ -190,5 +199,15 @@ namespace Squidex.Domain.Apps.Entities.Backup } } } + + private async Task ReadAsync() + { + await persistence.ReadAsync(); + } + + private async Task WriteAsync() + { + await persistence.WriteSnapshotAsync(state); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index b6840a584..677961d26 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -115,7 +115,7 @@ namespace Squidex.Domain.Apps.Entities.Backup await CleanupArchiveAsync(job); await CleanupBackupAsync(job); - job.IsFailed = true; + job.Status = JobStatus.Failed; await WriteAsync(); } @@ -164,7 +164,12 @@ namespace Squidex.Domain.Apps.Entities.Backup throw new DomainException($"You cannot have more than {MaxBackups} backups."); } - var job = new BackupStateJob { Id = Guid.NewGuid(), Started = clock.GetCurrentInstant() }; + var job = new BackupStateJob + { + Id = Guid.NewGuid(), + Started = clock.GetCurrentInstant(), + Status = JobStatus.Started + }; currentTask = new CancellationTokenSource(); currentJob = job; @@ -195,14 +200,7 @@ namespace Squidex.Domain.Apps.Entities.Backup job.HandledEvents = writer.WrittenEvents; job.HandledAssets = writer.WrittenAttachments; - var now = clock.GetCurrentInstant(); - - if ((now - lastTimestamp) >= UpdateDuration) - { - lastTimestamp = now; - - await WriteAsync(); - } + lastTimestamp = await WritePeriodically(lastTimestamp); }, SquidexHeaders.AppId, appId.ToString(), null, currentTask.Token); foreach (var handler in handlers) @@ -230,7 +228,7 @@ namespace Squidex.Domain.Apps.Entities.Backup .WriteProperty("status", "failed") .WriteProperty("backupId", job.Id.ToString())); - job.IsFailed = true; + job.Status = JobStatus.Failed; } finally { @@ -245,6 +243,20 @@ namespace Squidex.Domain.Apps.Entities.Backup } } + private async Task WritePeriodically(Instant lastTimestamp) + { + var now = clock.GetCurrentInstant(); + + if (ShouldUpdate(lastTimestamp, now)) + { + lastTimestamp = now; + + await WriteAsync(); + } + + return lastTimestamp; + } + public async Task DeleteAsync(Guid id) { var job = state.Jobs.FirstOrDefault(x => x.Id == id); @@ -274,6 +286,11 @@ namespace Squidex.Domain.Apps.Entities.Backup return J.AsTask(state.Jobs.OfType().ToList()); } + private static bool ShouldUpdate(Instant lastTimestamp, Instant now) + { + return (now - lastTimestamp) >= UpdateDuration; + } + private bool IsRunning() { return state.Jobs.Any(x => !x.Stopped.HasValue); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs index b3df4d5d1..0cf558335 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs @@ -14,6 +14,8 @@ namespace Squidex.Domain.Apps.Entities.Backup { public abstract class BackupHandler { + public abstract string Name { get; } + public virtual Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { return TaskHelper.Done; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs index 2de75b269..731cc92e2 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs @@ -10,6 +10,7 @@ using System.IO; using System.IO.Compression; using System.Threading.Tasks; using Newtonsoft.Json; +using Squidex.Domain.Apps.Entities.Backup.Archive; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; @@ -17,8 +18,6 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class BackupReader : DisposableObjectBase { - private const int MaxEventsPerFolder = 1000; - private const int MaxAttachmentFolders = 1000; private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); private readonly ZipArchive archive; private int readEvents; @@ -52,9 +51,7 @@ namespace Squidex.Domain.Apps.Entities.Backup Guard.NotNullOrEmpty(name, nameof(name)); Guard.NotNull(handler, nameof(handler)); - var attachmentFolder = Math.Abs(name.GetHashCode() % MaxAttachmentFolders); - var attachmentPath = $"attachments/{attachmentFolder}/{name}"; - var attachmentEntry = archive.GetEntry(attachmentPath); + var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name)); if (attachmentEntry == null) { @@ -75,9 +72,7 @@ namespace Squidex.Domain.Apps.Entities.Backup while (true) { - var eventFolder = readEvents / MaxEventsPerFolder; - var eventPath = $"events/{eventFolder}/{readEvents}.json"; - var eventEntry = archive.GetEntry(eventPath); + var eventEntry = archive.GetEntry(ArchiveHelper.GetEventPath(readEvents)); if (eventEntry == null) { diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupRestoreException.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupRestoreException.cs new file mode 100644 index 000000000..f7fec5453 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupRestoreException.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Runtime.Serialization; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + [Serializable] + public class BackupRestoreException : Exception + { + public BackupRestoreException(string message) + : base(message) + { + } + + public BackupRestoreException(string message, Exception inner) + : base(message, inner) + { + } + + protected BackupRestoreException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs index d8951d403..5f424036c 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs @@ -10,6 +10,7 @@ using System.IO; using System.IO.Compression; using System.Threading.Tasks; using Newtonsoft.Json; +using Squidex.Domain.Apps.Entities.Backup.Archive; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; @@ -17,8 +18,6 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class BackupWriter : DisposableObjectBase { - private const int MaxEventsPerFolder = 1000; - private const int MaxAttachmentFolders = 1000; private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); private readonly ZipArchive archive; private int writtenEvents; @@ -52,9 +51,7 @@ namespace Squidex.Domain.Apps.Entities.Backup Guard.NotNullOrEmpty(name, nameof(name)); Guard.NotNull(handler, nameof(handler)); - var attachmentFolder = Math.Abs(name.GetHashCode() % MaxAttachmentFolders); - var attachmentPath = $"attachments/{attachmentFolder}/{name}"; - var attachmentEntry = archive.CreateEntry(attachmentPath); + var attachmentEntry = archive.CreateEntry(ArchiveHelper.GetAttachmentPath(name)); using (var stream = attachmentEntry.Open()) { @@ -66,9 +63,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public void WriteEvent(StoredEvent storedEvent) { - var eventFolder = writtenEvents / MaxEventsPerFolder; - var eventPath = $"events/{eventFolder}/{writtenEvents}.json"; - var eventEntry = archive.GetEntry(eventPath) ?? archive.CreateEntry(eventPath); + var eventEntry = archive.CreateEntry(ArchiveHelper.GetEventPath(writtenEvents)); using (var stream = eventEntry.Open()) { diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Helpers/ArchiveHelper.cs b/src/Squidex.Domain.Apps.Entities/Backup/Helpers/ArchiveHelper.cs new file mode 100644 index 000000000..ed0c5778c --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Helpers/ArchiveHelper.cs @@ -0,0 +1,50 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; + +namespace Squidex.Domain.Apps.Entities.Backup.Archive +{ + public static class ArchiveHelper + { + private const int MaxAttachmentFolders = 1000; + private const int MaxEventsPerFolder = 1000; + + public static string GetAttachmentPath(string name) + { + name = name.ToLowerInvariant(); + + var attachmentFolder = SimpleHash(name) % MaxAttachmentFolders; + var attachmentPath = $"attachments/{attachmentFolder}/{name}"; + + return attachmentPath; + } + + public static string GetEventPath(int index) + { + var eventFolder = index / MaxEventsPerFolder; + var eventPath = $"events/{eventFolder}/{index}.json"; + + return eventPath; + } + + private static int SimpleHash(string value) + { + var hash = 17; + + foreach (char c in value) + { + unchecked + { + hash = (hash * 23) + c.GetHashCode(); + } + } + + return Math.Abs(hash); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Helpers/Downloader.cs b/src/Squidex.Domain.Apps.Entities/Backup/Helpers/Downloader.cs new file mode 100644 index 000000000..f002efa81 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Helpers/Downloader.cs @@ -0,0 +1,66 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup.Helpers +{ + public static class Downloader + { + public static async Task DownloadAsync(this IBackupArchiveLocation backupArchiveLocation, Uri url, Guid id) + { + HttpResponseMessage response = null; + try + { + using (var client = new HttpClient()) + { + response = await client.GetAsync(url); + response.EnsureSuccessStatusCode(); + + using (var sourceStream = await response.Content.ReadAsStreamAsync()) + { + using (var targetStream = await backupArchiveLocation.OpenStreamAsync(id)) + { + await sourceStream.CopyToAsync(targetStream); + } + } + } + } + catch (HttpRequestException ex) + { + throw new BackupRestoreException($"Cannot download the archive. Got status code: {response?.StatusCode}.", ex); + } + } + + public static async Task OpenArchiveAsync(this IBackupArchiveLocation backupArchiveLocation, Guid id) + { + Stream stream = null; + + try + { + stream = await backupArchiveLocation.OpenStreamAsync(id); + + return new BackupReader(stream); + } + catch (IOException) + { + stream?.Dispose(); + + throw new BackupRestoreException("The backup archive is correupt and cannot be opened."); + } + catch (Exception) + { + stream?.Dispose(); + + throw; + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs index 4142fd5e0..56fe0a19e 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs @@ -22,6 +22,6 @@ namespace Squidex.Domain.Apps.Entities.Backup int HandledAssets { get; } - bool IsFailed { get; } + JobStatus Status { get; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs index 219169dd3..f38df5629 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs @@ -7,15 +7,15 @@ using System; using System.Threading.Tasks; -using Squidex.Infrastructure; +using Orleans; using Squidex.Infrastructure.Orleans; namespace Squidex.Domain.Apps.Entities.Backup { - public interface IRestoreGrain + public interface IRestoreGrain : IGrainWithStringKey { - Task RestoreAsync(Uri url, RefToken user); + Task RestoreAsync(Uri url); - Task> GetStateAsync(); + Task> GetJobAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs index b3714d15d..8d0db185c 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs @@ -6,6 +6,7 @@ // ========================================================================== using System; +using System.Collections.Generic; using NodaTime; namespace Squidex.Domain.Apps.Entities.Backup @@ -16,8 +17,10 @@ namespace Squidex.Domain.Apps.Entities.Backup Instant Started { get; } - bool IsFailed { get; } + Instant? Stopped { get; } - string Status { get; } + List Log { get; } + + JobStatus Status { get; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/JobStatus.cs b/src/Squidex.Domain.Apps.Entities/Backup/JobStatus.cs new file mode 100644 index 000000000..26f6f541c --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/JobStatus.cs @@ -0,0 +1,17 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public enum JobStatus + { + Created, + Started, + Completed, + Failed + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index 2be9eef36..fe50bca91 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -7,10 +7,10 @@ using System; using System.Collections.Generic; -using System.Net.Http; using System.Threading.Tasks; using NodaTime; using Orleans; +using Squidex.Domain.Apps.Entities.Backup.Helpers; using Squidex.Domain.Apps.Entities.Backup.State; using Squidex.Domain.Apps.Events; using Squidex.Domain.Apps.Events.Apps; @@ -26,7 +26,6 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class RestoreGrain : GrainOfString, IRestoreGrain { - private static readonly Duration UpdateDuration = Duration.FromSeconds(1); private readonly IClock clock; private readonly IAssetStore assetStore; private readonly IEventDataFormatter eventDataFormatter; @@ -37,6 +36,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IBackupArchiveLocation backupArchiveLocation; private readonly IStore store; private readonly IEnumerable handlers; + private RefToken actor; private RestoreState state = new RestoreState(); private IPersistence persistence; @@ -76,36 +76,56 @@ namespace Squidex.Domain.Apps.Entities.Backup public override async Task OnActivateAsync(string key) { + actor = new RefToken("subject", key); + persistence = store.WithSnapshots(GetType(), key, s => state = s); - await persistence.ReadAsync(); + await ReadAsync(); - await CleanupAsync(); + RecoverAfterRestart(); } - public Task RestoreAsync(Uri url, RefToken user) + private void RecoverAfterRestart() { - if (state.Job != null) - { - throw new DomainException("A restore operation is already running."); - } + RecoverAfterRestartAsync().Forget(); + } - state.Job = new RestoreStateJob { Started = clock.GetCurrentInstant(), Uri = url, User = user }; + private async Task RecoverAfterRestartAsync() + { + if (state.Job?.Status == JobStatus.Started) + { + Log("Failed due application restart"); - return ProcessAsync(); + await CleanupAsync(); + await WriteAsync(); + } } - private async Task CleanupAsync() + public Task RestoreAsync(Uri url) { - if (state.Job != null) + Guard.NotNull(url, nameof(url)); + + if (state.Job?.Status == JobStatus.Started) + { + throw new DomainException("A restore operation is already running."); + } + + state.Job = new RestoreStateJob { - state.Job.Status = "Failed due application restart"; - state.Job.IsFailed = true; + Id = Guid.NewGuid(), + Started = clock.GetCurrentInstant(), + Status = JobStatus.Started, + Uri = url + }; - TryCleanup(); + Process(); - await persistence.WriteSnapshotAsync(state); - } + return TaskHelper.Done; + } + + private void Process() + { + ProcessAsync().Forget(); } private async Task ProcessAsync() @@ -119,49 +139,40 @@ namespace Squidex.Domain.Apps.Entities.Backup .WriteProperty("status", "started") .WriteProperty("url", state.Job.Uri.ToString())); - state.Job.Status = "Downloading Backup"; - using (Profiler.Trace("Download")) { await DownloadAsync(); } - state.Job.Status = "Downloaded Backup"; - - using (var stream = await backupArchiveLocation.OpenStreamAsync(state.Job.Id)) + using (var reader = await backupArchiveLocation.OpenArchiveAsync(state.Job.Id)) { - using (var reader = new BackupReader(stream)) + using (Profiler.Trace("ReadEvents")) + { + await ReadEventsAsync(reader); + } + + foreach (var handler in handlers) { - using (Profiler.Trace("ReadEvents")) + using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RestoreAsync))) { - await ReadEventsAsync(reader); + await handler.RestoreAsync(state.Job.AppId, reader); } - state.Job.Status = "Events read"; + Log($"Restored {handler.Name}"); + } - foreach (var handler in handlers) + foreach (var handler in handlers) + { + using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.CompleteRestoreAsync))) { - using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RestoreAsync))) - { - await handler.RestoreAsync(state.Job.AppId, reader); - } - - state.Job.Status = $"{handler} Processed"; + await handler.CompleteRestoreAsync(state.Job.AppId, reader); } - foreach (var handler in handlers) - { - using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.CompleteRestoreAsync))) - { - await handler.CompleteRestoreAsync(state.Job.AppId, reader); - } - - state.Job.Status = $"{handler} Completed"; - } + Log($"Completed {handler.Name}"); } } - state.Job = null; + state.Job.Status = JobStatus.Failed; log.LogInformation(w => { @@ -174,17 +185,25 @@ namespace Squidex.Domain.Apps.Entities.Backup } catch (Exception ex) { - state.Job.IsFailed = true; + if (ex is BackupRestoreException backupException) + { + Log(backupException.Message); + } + else + { + Log("Failed with internal error"); + } - if (state.Job.AppId != Guid.Empty) + try { - foreach (var handler in handlers) - { - await handler.CleanupRestoreAsync(state.Job.AppId, ex); - } + await CleanupAsync(ex); + } + catch (Exception ex2) + { + ex = ex2; } - TryCleanup(); + state.Job.Status = JobStatus.Failed; log.LogError(ex, w => { @@ -197,37 +216,46 @@ namespace Squidex.Domain.Apps.Entities.Backup } finally { - await persistence.WriteSnapshotAsync(state); + await WriteAsync(); } } } - private async Task DownloadAsync() + private async Task CleanupAsync(Exception exception = null) { - using (var client = new HttpClient()) + await backupArchiveLocation.DeleteArchiveAsync(state.Job.Id); + + if (state.Job.AppId != Guid.Empty) { - using (var sourceStream = await client.GetStreamAsync(state.Job.Uri.ToString())) + foreach (var handler in handlers) { - using (var targetStream = await backupArchiveLocation.OpenStreamAsync(state.Job.Id)) - { - await sourceStream.CopyToAsync(targetStream); - } + await handler.CleanupRestoreAsync(state.Job.AppId, exception); } + + await appCleaner.EnqueueAppAsync(state.Job.AppId); } } + private async Task DownloadAsync() + { + Log("Downloading Backup"); + + await backupArchiveLocation.DownloadAsync(state.Job.Uri, state.Job.Id); + + Log("Downloaded Backup"); + } + private async Task ReadEventsAsync(BackupReader reader) { await reader.ReadEventsAsync(async (storedEvent) => { - var eventData = storedEvent.Data; - var eventParsed = eventDataFormatter.Parse(eventData); + var @event = eventDataFormatter.Parse(storedEvent.Data); - if (eventParsed.Payload is SquidexEvent squidexEvent) + if (@event.Payload is SquidexEvent squidexEvent) { - squidexEvent.Actor = state.Job.User; + squidexEvent.Actor = actor; } - else if (eventParsed.Payload is AppCreated appCreated) + else if (@event.Payload is AppCreated appCreated) { state.Job.AppId = appCreated.AppId.Id; @@ -236,13 +264,15 @@ namespace Squidex.Domain.Apps.Entities.Backup foreach (var handler in handlers) { - await handler.RestoreEventAsync(eventParsed, state.Job.AppId, reader); + await handler.RestoreEventAsync(@event, state.Job.AppId, reader); } await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, new List { storedEvent.Data }); - state.Job.Status = $"Handled event {reader.ReadEvents} events and {reader.ReadAttachments} attachments"; + Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments."); }); + + Log("Reading events completed."); } private async Task CheckCleanupStatus() @@ -253,24 +283,31 @@ namespace Squidex.Domain.Apps.Entities.Backup if (status == CleanerStatus.Cleaning) { - throw new DomainException("The app is removed in the background."); + throw new BackupRestoreException("The app is removed in the background."); } if (status == CleanerStatus.Cleaning) { - throw new DomainException("The app could not be cleaned."); + throw new BackupRestoreException("The app could not be cleaned."); } } - private void TryCleanup() + private void Log(string message) { - if (state.Job.AppId != Guid.Empty) - { - appCleaner.EnqueueAppAsync(state.Job.AppId).Forget(); - } + state.Job.Log.Add($"{clock.GetCurrentInstant()}: {message}"); + } + + private async Task ReadAsync() + { + await persistence.ReadAsync(); + } + + private async Task WriteAsync() + { + await persistence.WriteSnapshotAsync(state); } - public Task> GetStateAsync() + public Task> GetJobAsync() { return Task.FromResult>(state.Job); } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs index 58c5d37b4..6da5e9dfa 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs @@ -29,6 +29,6 @@ namespace Squidex.Domain.Apps.Entities.Backup.State public int HandledAssets { get; set; } [JsonProperty] - public bool IsFailed { get; set; } + public JobStatus Status { get; set; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs index 9c794bae3..1eac3612b 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs @@ -6,23 +6,20 @@ // ========================================================================== using System; +using System.Collections.Generic; using Newtonsoft.Json; using NodaTime; -using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Backup.State { public sealed class RestoreStateJob : IRestoreJob { [JsonProperty] - public Guid Id { get; set; } = Guid.NewGuid(); + public Guid Id { get; set; } [JsonProperty] public Guid AppId { get; set; } - [JsonProperty] - public RefToken User { get; set; } - [JsonProperty] public Uri Uri { get; set; } @@ -30,9 +27,12 @@ namespace Squidex.Domain.Apps.Entities.Backup.State public Instant Started { get; set; } [JsonProperty] - public string Status { get; set; } + public Instant? Stopped { get; set; } + + [JsonProperty] + public List Log { get; set; } [JsonProperty] - public bool IsFailed { get; set; } + public JobStatus Status { get; set; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs b/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs index b710a46ac..7fb773874 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs @@ -18,7 +18,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { var tempFile = GetTempFile(backupId); - return Task.FromResult(new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite)); + return Task.FromResult(new FileStream(tempFile, FileMode.OpenOrCreate, FileAccess.ReadWrite)); } public Task DeleteArchiveAsync(Guid backupId) @@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private static string GetTempFile(Guid backupId) { - return Path.Combine(Path.GetTempPath(), backupId.ToString()); + return Path.Combine(Path.GetTempPath(), backupId.ToString() + ".zip"); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs index f4d837206..1e9b76654 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs @@ -55,7 +55,7 @@ namespace Squidex.Domain.Apps.Entities.Tags { state.Tags = tags; - return persistence.DeleteAsync(); + return persistence.WriteSnapshotAsync(state); } public async Task> NormalizeTagsAsync(HashSet names, HashSet ids) diff --git a/src/Squidex.Infrastructure/Tasks/TaskExtensions.cs b/src/Squidex.Infrastructure/Tasks/TaskExtensions.cs index 8d5f8a548..416808ba0 100644 --- a/src/Squidex.Infrastructure/Tasks/TaskExtensions.cs +++ b/src/Squidex.Infrastructure/Tasks/TaskExtensions.cs @@ -6,14 +6,30 @@ // ========================================================================== using System; +using System.Threading; using System.Threading.Tasks; namespace Squidex.Infrastructure.Tasks { public static class TaskExtensions { + private static readonly Action IgnoreTaskContinuation = t => { var ignored = t.Exception; }; + public static void Forget(this Task task) { + if (task.IsCompleted) + { + var ignored = task.Exception; + } + else + { + task.ContinueWith( + IgnoreTaskContinuation, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } } public static Func ToDefault(this Action action) diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs new file mode 100644 index 000000000..a419a5f33 --- /dev/null +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs @@ -0,0 +1,45 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.ComponentModel.DataAnnotations; +using NodaTime; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Infrastructure.Reflection; + +namespace Squidex.Areas.Api.Controllers.Backups.Models +{ + public sealed class RestoreJobDto + { + /// + /// The uri to load from. + /// + [Required] + public Uri Uri { get; set; } + + /// + /// The status text. + /// + [Required] + public string Status { get; set; } + + /// + /// Indicates when the restore operation has been started. + /// + public Instant Started { get; set; } + + /// + /// Indicates if the restore has failed. + /// + public bool IsFailed { get; set; } + + public static RestoreJobDto FromJob(IRestoreJob job) + { + return SimpleMapper.Map(job, new RestoreJobDto()); + } + } +} diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs new file mode 100644 index 000000000..50dd8ec55 --- /dev/null +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs @@ -0,0 +1,21 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.ComponentModel.DataAnnotations; + +namespace Squidex.Areas.Api.Controllers.Backups.Models +{ + public sealed class RestoreRequest + { + /// + /// The url to the restore file. + /// + [Required] + public Uri Url { get; set; } + } +} diff --git a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs new file mode 100644 index 000000000..8c133ff79 --- /dev/null +++ b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs @@ -0,0 +1,86 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using NSwag.Annotations; +using Orleans; +using Squidex.Areas.Api.Controllers.Backups.Models; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Infrastructure.Commands; +using Squidex.Infrastructure.Security; +using Squidex.Infrastructure.Tasks; +using Squidex.Pipeline; + +namespace Squidex.Areas.Api.Controllers.Backups +{ + /// + /// Restores backups. + /// + [ApiAuthorize] + [ApiExceptionFilter] + [ApiModelValidation(true)] + [SwaggerTag(nameof(Backups))] + public class RestoreController : ApiController + { + private readonly IGrainFactory grainFactory; + + public RestoreController(ICommandBus commandBus, IGrainFactory grainFactory) + : base(commandBus) + { + this.grainFactory = grainFactory; + } + + /// + /// Get the restore jobs. + /// + /// + /// 200 => Restore job returned. + /// 404 => App not found. + /// + [HttpGet] + [Route("apps/restore/")] + [ProducesResponseType(typeof(RestoreJobDto), 200)] + [ApiCosts(0)] + public async Task GetJob() + { + var restoreGrain = grainFactory.GetGrain(User.OpenIdSubject()); + + var job = await restoreGrain.GetJobAsync(); + + if (job.Value == null) + { + return NotFound(); + } + + var jobs = await restoreGrain.GetJobAsync(); + + var response = RestoreJobDto.FromJob(job.Value); + + return Ok(response); + } + + /// + /// Start a new restore job. + /// + /// The request object. + /// + /// 204 => Backup started. + /// + [HttpPost] + [Route("apps/restore/")] + [ApiCosts(0)] + public async Task PostRestore([FromBody] RestoreRequest request) + { + var restoreGrain = grainFactory.GetGrain(User.OpenIdSubject()); + + await restoreGrain.RestoreAsync(request.Url); + + return NoContent(); + } + } +} diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index b690a4d1b..385fb4e4e 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -185,12 +185,23 @@ namespace Squidex.Config.Domain private static void AddBackupHandlers(this IServiceCollection services) { - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); + services.AddTransientAs() + .As(); + + services.AddTransientAs() + .As(); + + services.AddTransientAs() + .As(); + + services.AddTransientAs() + .As(); + + services.AddTransientAs() + .As(); + + services.AddTransientAs() + .As(); } public static void AddMyMigrationServices(this IServiceCollection services) @@ -198,6 +209,9 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .AsSelf(); + services.AddTransientAs() + .AsSelf(); + services.AddTransientAs() .As(); @@ -227,9 +241,6 @@ namespace Squidex.Config.Domain services.AddTransientAs() .As(); - - services.AddTransientAs() - .AsSelf(); } } } diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.html b/src/Squidex/app/features/apps/pages/apps-page.component.html index 461321025..1d29c5506 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.html +++ b/src/Squidex/app/features/apps/pages/apps-page.component.html @@ -1,6 +1,6 @@  -
+

Hi {{authState.user?.displayName}}

@@ -8,71 +8,115 @@
- -
-
-

You are not collaborating to any app yet

-
- -
-
-

{{app.name}}

- -
- Edit +
+
+
+ +
+
+

You are not collaborating to any app yet

+
+ +
+
+

{{app.name}}

+ +
+ Edit +
+
+
-
-
-
- - -
-
-
-
- -
- -

New App

- -
- Create a new blank app without content and schemas. -
-
-
- -
-
-
- -
- -

New Blog Sample

- -
-
Start with our ready to use blog.
-
- Sample Code: C# + + +
+
+
+
+ +
+ +

New App

+ +
+ Create a new blank app without content and schemas. +
+
+
+ +
+
+
+ +
+ +

New Blog Sample

+ +
+
Start with our ready to use blog.
+
+ Sample Code: C# +
+
+
+
+ +
+
+
+ +
+ +

New Profile Sample

+ +
+
Create your profile page.
+
+ Sample Code: C# +
+
+
- -
-
-
- +
+

Restore Backup

+ + +
+
+
+
+
+ +
+
+ +
+
+ +
+

Restore

+
+
+
+
+ Status: {{job.status}} +
+ -

New Profile Sample

- -
-
Create your profile page.
-
- Sample Code: C# +
+
+
+ +
+
+
-
+
diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.scss b/src/Squidex/app/features/apps/pages/apps-page.component.scss index b789d434d..835a60c5a 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.scss +++ b/src/Squidex/app/features/apps/pages/apps-page.component.scss @@ -12,11 +12,22 @@ padding-top: 2rem; padding-right: 1.25rem; padding-bottom: 0; - padding-left: $size-sidebar-width + .25rem; display: block; } } +.col-left { + padding-left: $size-sidebar-width + .25rem; +} + +.col-right { + border-left: 1px solid darken($color-border, 3%); + min-height: 200px; + min-width: 300px; + max-width: 300px; + padding: 0 2rem 0 1rem; +} + .card { & { margin-right: 1rem; @@ -79,4 +90,37 @@ text-decoration: none; } } +} + +$cicle-size: 1.5rem; + +.restore { + &-card { + float: none; + } + + &-status { + & { + @include circle($cicle-size); + line-height: $cicle-size + .1rem; + text-align: center; + font-size: .6 * $cicle-size; + font-weight: normal; + background: $color-border; + color: $color-dark-foreground; + vertical-align: middle; + } + + &-pending { + color: inherit; + } + + &-failed { + background: $color-theme-error; + } + + &-success { + background: $color-theme-green; + } + } } \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.ts b/src/Squidex/app/features/apps/pages/apps-page.component.ts index c081ccb84..1414375b7 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.ts +++ b/src/Squidex/app/features/apps/pages/apps-page.component.ts @@ -5,15 +5,21 @@ * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. */ -import { Component, OnInit } from '@angular/core'; -import { take } from 'rxjs/operators'; +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { FormBuilder } from '@angular/forms'; +import { Subscription, timer } from 'rxjs'; +import { switchMap, take } from 'rxjs/operators'; import { AppsState, AuthService, + BackupsService, DialogModel, + DialogService, ModalModel, - OnboardingService + OnboardingService, + RestoreDto, + RestoreForm } from '@app/shared'; @Component({ @@ -21,20 +27,38 @@ import { styleUrls: ['./apps-page.component.scss'], templateUrl: './apps-page.component.html' }) -export class AppsPageComponent implements OnInit { +export class AppsPageComponent implements OnDestroy, OnInit { + private timerSubscription: Subscription; + public addAppDialog = new DialogModel(); public addAppTemplate = ''; + public restoreJob: RestoreDto | null; + public restoreForm = new RestoreForm(this.formBuilder); + public onboardingModal = new ModalModel(); constructor( public readonly appsState: AppsState, public readonly authState: AuthService, + private readonly backupsService: BackupsService, + private readonly dialogs: DialogService, + private readonly formBuilder: FormBuilder, private readonly onboardingService: OnboardingService ) { } + public ngOnDestroy() { + this.timerSubscription.unsubscribe(); + } + public ngOnInit() { + this.timerSubscription = + timer(0, 3000).pipe(switchMap(t => this.backupsService.getRestore())) + .subscribe(dto => { + this.restoreJob = dto; + }); + this.appsState.apps.pipe( take(1)) .subscribe(apps => { @@ -45,6 +69,21 @@ export class AppsPageComponent implements OnInit { }); } + public restore() { + const value = this.restoreForm.submit(); + + if (value) { + this.restoreForm.submitCompleted({}); + + this.backupsService.postRestore(value.url) + .subscribe(() => { + this.dialogs.notifyInfo('Restore started, it can take several minutes to complete.'); + }, error => { + this.dialogs.notifyError(error); + }); + } + } + public createNewApp(template: string) { this.addAppTemplate = template; this.addAppDialog.show(); diff --git a/src/Squidex/app/features/settings/pages/backups/backups-page.component.html b/src/Squidex/app/features/settings/pages/backups/backups-page.component.html index d413f2ba4..17f3bf72b 100644 --- a/src/Squidex/app/features/settings/pages/backups/backups-page.component.html +++ b/src/Squidex/app/features/settings/pages/backups/backups-page.component.html @@ -33,7 +33,7 @@
-
+
diff --git a/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss b/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss index 80a10e112..7d14e7dd0 100644 --- a/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss +++ b/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss @@ -1,14 +1,14 @@ @import '_vars'; @import '_mixins'; -$cicle-size: 2.8rem; +$circle-size: 2.8rem; .backup-status { & { - @include circle($cicle-size); - line-height: $cicle-size + .1rem; + @include circle($circle-size); + line-height: $circle-size + .1rem; text-align: center; - font-size: .4 * $cicle-size; + font-size: .4 * $circle-size; font-weight: normal; background: $color-border; color: $color-dark-foreground; diff --git a/src/Squidex/app/features/settings/pages/languages/language.component.html b/src/Squidex/app/features/settings/pages/languages/language.component.html index fb6a2f7a8..7567144f8 100644 --- a/src/Squidex/app/features/settings/pages/languages/language.component.html +++ b/src/Squidex/app/features/settings/pages/languages/language.component.html @@ -40,7 +40,7 @@
-
+
{{language.englishName}}
diff --git a/src/Squidex/app/shared/components/history-list.component.scss b/src/Squidex/app/shared/components/history-list.component.scss index 0eabbf8e9..17c4744e7 100644 --- a/src/Squidex/app/shared/components/history-list.component.scss +++ b/src/Squidex/app/shared/components/history-list.component.scss @@ -15,6 +15,10 @@ margin-top: .25rem; } +.user-ref { + padding-right: .25rem; +} + .event { & { color: $color-history; diff --git a/src/Squidex/app/shared/internal.ts b/src/Squidex/app/shared/internal.ts index 02a02f1fa..51fb48eb8 100644 --- a/src/Squidex/app/shared/internal.ts +++ b/src/Squidex/app/shared/internal.ts @@ -44,6 +44,7 @@ export * from './state/apps.forms'; export * from './state/apps.state'; export * from './state/assets.forms'; export * from './state/assets.state'; +export * from './state/backups.forms'; export * from './state/backups.state'; export * from './state/clients.forms'; export * from './state/clients.state'; diff --git a/src/Squidex/app/shared/services/backups.service.ts b/src/Squidex/app/shared/services/backups.service.ts index d29099d8f..563b809e7 100644 --- a/src/Squidex/app/shared/services/backups.service.ts +++ b/src/Squidex/app/shared/services/backups.service.ts @@ -5,17 +5,18 @@ * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. */ -import { HttpClient } from '@angular/common/http'; +import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { Observable } from 'rxjs'; -import { map, tap } from 'rxjs/operators'; +import { Observable, of, throwError } from 'rxjs'; +import { catchError, map, tap } from 'rxjs/operators'; import { AnalyticsService, ApiUrlConfig, DateTime, Model, - pretifyError + pretifyError, + Types } from '@app/framework'; export class BackupDto extends Model { @@ -35,6 +36,16 @@ export class BackupDto extends Model { } } +export class RestoreDto { + constructor( + public readonly started: DateTime, + public readonly status: string, + public readonly url: string, + public readonly isFailed: boolean + ) { + } +} + @Injectable() export class BackupsService { constructor( @@ -64,6 +75,29 @@ export class BackupsService { pretifyError('Failed to load backups.')); } + public getRestore(): Observable { + const url = this.apiUrl.buildUrl(`api/apps/restore`); + + return this.http.get(url).pipe( + map(response => { + const body: any = response; + + return new RestoreDto( + DateTime.parseISO_UTC(body.started), + body.status, + body.url, + body.isFailed); + }), + catchError(error => { + if (Types.is(error, HttpErrorResponse) && error.status === 404) { + return of(null); + } else { + return throwError(error); + } + }), + pretifyError('Failed to load backups.')); + } + public postBackup(appName: string): Observable { const url = this.apiUrl.buildUrl(`api/apps/${appName}/backups`); @@ -74,6 +108,16 @@ export class BackupsService { pretifyError('Failed to start backup.')); } + public postRestore(downloadUrl: string): Observable { + const url = this.apiUrl.buildUrl(`api/apps/restore`); + + return this.http.post(url, { url: downloadUrl }).pipe( + tap(() => { + this.analytics.trackEvent('Restore', 'Started'); + }), + pretifyError('Failed to start restore.')); + } + public deleteBackup(appName: string, id: string): Observable { const url = this.apiUrl.buildUrl(`api/apps/${appName}/backups/${id}`); diff --git a/src/Squidex/app/shared/state/backups.forms.ts b/src/Squidex/app/shared/state/backups.forms.ts new file mode 100644 index 000000000..cbdb82d5b --- /dev/null +++ b/src/Squidex/app/shared/state/backups.forms.ts @@ -0,0 +1,26 @@ +/* + * Squidex Headless CMS + * + * @license + * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. + */ + +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { map, startWith } from 'rxjs/operators'; + +import { Form } from '@app/framework'; + +export class RestoreForm extends Form { + public hasNoUrl = + this.form.controls['url'].valueChanges.pipe(startWith(null), map(x => !x)); + + constructor(formBuilder: FormBuilder) { + super(formBuilder.group({ + url: [null, + [ + Validators.required + ] + ] + })); + } +} \ No newline at end of file From d3481917b897b937dc90fa028e01c426b59639fb Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 30 Jul 2018 19:52:48 +0200 Subject: [PATCH 14/23] Separate page for restore and refactoring. --- .../Apps/BackupApps.cs | 4 +- .../Assets/BackupAssets.cs | 2 + .../Backup/BackupGrain.cs | 79 ++------- .../Backup/BackupHandler.cs | 2 +- .../Backup/Helpers/Safe.cs | 62 +++++++ .../Backup/IBackupGrain.cs | 2 - .../Backup/RestoreGrain.cs | 83 +++++---- .../Backup/State/RestoreStateJob.cs | 2 +- .../Contents/BackupContents.cs | 2 + .../History/BackupHistory.cs | 2 + .../Rules/BackupRules.cs | 2 + .../Schemas/BackupSchemas.cs | 2 + .../Backups/Models/BackupJobDto.cs | 4 +- .../Backups/Models/RestoreJobDto.cs | 16 +- src/Squidex/app/features/apps/declarations.ts | 3 +- src/Squidex/app/features/apps/module.ts | 9 +- .../apps/pages/apps-page.component.html | 162 +++++++----------- .../apps/pages/apps-page.component.scss | 46 +---- .../apps/pages/apps-page.component.ts | 47 +---- .../apps/pages/restore-page.component.html | 55 ++++++ .../apps/pages/restore-page.component.scss | 59 +++++++ .../apps/pages/restore-page.component.ts | 66 +++++++ .../events/rule-events-page.component.scss | 1 + .../pages/backups/backups-page.component.html | 6 +- .../shared/services/backups.service.spec.ts | 8 +- .../app/shared/services/backups.service.ts | 14 +- .../app/shared/state/backups.state.spec.ts | 4 +- 27 files changed, 435 insertions(+), 309 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/Helpers/Safe.cs create mode 100644 src/Squidex/app/features/apps/pages/restore-page.component.html create mode 100644 src/Squidex/app/features/apps/pages/restore-page.component.scss create mode 100644 src/Squidex/app/features/apps/pages/restore-page.component.ts diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs index 3c3cfc36f..34d2c9bcd 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -27,6 +27,8 @@ namespace Squidex.Domain.Apps.Entities.Apps private bool isReserved; private AppCreated appCreated; + public override string Name { get; } = "Apps"; + public BackupApps(IStore store, IGrainFactory grainFactory) : base(store) { @@ -78,7 +80,7 @@ namespace Squidex.Domain.Apps.Entities.Apps } } - public override async Task CleanupRestoreAsync(Guid appId, Exception exception) + public override async Task CleanupRestoreAsync(Guid appId) { if (isReserved) { diff --git a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index e9157981f..3061a39af 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -32,6 +32,8 @@ namespace Squidex.Domain.Apps.Entities.Assets private readonly ITagService tagService; private readonly IEventDataFormatter eventDataFormatter; + public override string Name { get; } = "Assets"; + public BackupAssets(IStore store, IEventDataFormatter eventDataFormatter, IAssetStore assetStore, diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index 677961d26..491c2b05f 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -12,6 +12,7 @@ using System.Threading; using System.Threading.Tasks; using NodaTime; using Orleans.Concurrency; +using Squidex.Domain.Apps.Entities.Backup.Helpers; using Squidex.Domain.Apps.Entities.Backup.State; using Squidex.Domain.Apps.Events; using Squidex.Infrastructure; @@ -20,6 +21,7 @@ using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Backup { @@ -78,33 +80,16 @@ namespace Squidex.Domain.Apps.Entities.Backup persistence = store.WithSnapshots(GetType(), key, s => state = s); await ReadAsync(); - await CleanupAsync(); - } - - public async Task ClearAsync() - { - foreach (var job in state.Jobs) - { - await CleanupArchiveAsync(job); - await CleanupBackupAsync(job); - } - state = new BackupState(); - - await persistence.DeleteAsync(); - } - - private async Task ReadAsync() - { - await persistence.ReadAsync(); + RecoverAfterRestart(); } - private async Task WriteAsync() + private void RecoverAfterRestart() { - await persistence.WriteSnapshotAsync(state); + RecoverAfterRestartAsync().Forget(); } - private async Task CleanupAsync() + private async Task RecoverAfterRestartAsync() { foreach (var job in state.Jobs) { @@ -112,8 +97,8 @@ namespace Squidex.Domain.Apps.Entities.Backup { job.Stopped = clock.GetCurrentInstant(); - await CleanupArchiveAsync(job); - await CleanupBackupAsync(job); + await Safe.DeleteAsync(backupArchiveLocation, job.Id, log); + await Safe.DeleteAsync(assetStore, job.Id, log); job.Status = JobStatus.Failed; @@ -122,36 +107,6 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - private async Task CleanupBackupAsync(BackupStateJob job) - { - try - { - await assetStore.DeleteAsync(job.Id.ToString(), 0, null); - } - catch (Exception ex) - { - log.LogError(ex, w => w - .WriteProperty("action", "deleteBackup") - .WriteProperty("status", "failed") - .WriteProperty("backupId", job.Id.ToString())); - } - } - - private async Task CleanupArchiveAsync(BackupStateJob job) - { - try - { - await backupArchiveLocation.DeleteArchiveAsync(job.Id); - } - catch (Exception ex) - { - log.LogError(ex, w => w - .WriteProperty("action", "deleteArchive") - .WriteProperty("status", "failed") - .WriteProperty("backupId", job.Id.ToString())); - } - } - public async Task RunAsync() { if (currentTask != null) @@ -220,6 +175,8 @@ namespace Squidex.Domain.Apps.Entities.Backup await assetStore.UploadAsync(job.Id.ToString(), 0, null, stream, currentTask.Token); } + + job.Status = JobStatus.Completed; } catch (Exception ex) { @@ -232,7 +189,7 @@ namespace Squidex.Domain.Apps.Entities.Backup } finally { - await CleanupArchiveAsync(job); + await Safe.DeleteAsync(backupArchiveLocation, job.Id, log); job.Stopped = clock.GetCurrentInstant(); @@ -247,7 +204,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { var now = clock.GetCurrentInstant(); - if (ShouldUpdate(lastTimestamp, now)) + if ((now - lastTimestamp) >= UpdateDuration) { lastTimestamp = now; @@ -272,8 +229,8 @@ namespace Squidex.Domain.Apps.Entities.Backup } else { - await CleanupArchiveAsync(job); - await CleanupBackupAsync(job); + await Safe.DeleteAsync(backupArchiveLocation, job.Id, log); + await Safe.DeleteAsync(assetStore, job.Id, log); state.Jobs.Remove(job); @@ -286,14 +243,14 @@ namespace Squidex.Domain.Apps.Entities.Backup return J.AsTask(state.Jobs.OfType().ToList()); } - private static bool ShouldUpdate(Instant lastTimestamp, Instant now) + private async Task ReadAsync() { - return (now - lastTimestamp) >= UpdateDuration; + await persistence.ReadAsync(); } - private bool IsRunning() + private async Task WriteAsync() { - return state.Jobs.Any(x => !x.Stopped.HasValue); + await persistence.WriteSnapshotAsync(state); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs index 0cf558335..d879e8639 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs @@ -36,7 +36,7 @@ namespace Squidex.Domain.Apps.Entities.Backup return TaskHelper.Done; } - public virtual Task CleanupRestoreAsync(Guid appId, Exception exception) + public virtual Task CleanupRestoreAsync(Guid appId) { return TaskHelper.Done; } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/Helpers/Safe.cs b/src/Squidex.Domain.Apps.Entities/Backup/Helpers/Safe.cs new file mode 100644 index 000000000..d599881b6 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/Helpers/Safe.cs @@ -0,0 +1,62 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.Log; + +namespace Squidex.Domain.Apps.Entities.Backup.Helpers +{ + public static class Safe + { + public static async Task DeleteAsync(IBackupArchiveLocation backupArchiveLocation, Guid id, ISemanticLog log) + { + try + { + await backupArchiveLocation.DeleteArchiveAsync(id); + } + catch (Exception ex) + { + log.LogError(ex, w => w + .WriteProperty("action", "deleteArchive") + .WriteProperty("status", "failed") + .WriteProperty("operationId", id.ToString())); + } + } + + public static async Task DeleteAsync(IAssetStore assetStore, Guid id, ISemanticLog log) + { + try + { + await assetStore.DeleteAsync(id.ToString(), 0, null); + } + catch (Exception ex) + { + log.LogError(ex, w => w + .WriteProperty("action", "deleteBackup") + .WriteProperty("status", "failed") + .WriteProperty("operationId", id.ToString())); + } + } + + public static async Task CleanupRestoreAsync(BackupHandler handler, Guid appId, Guid id, ISemanticLog log) + { + try + { + await handler.CleanupRestoreAsync(appId); + } + catch (Exception ex) + { + log.LogError(ex, w => w + .WriteProperty("action", "cleanupRestore") + .WriteProperty("status", "failed") + .WriteProperty("operationId", id.ToString())); + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs index aa48fdbd4..21f66e2ff 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs @@ -19,8 +19,6 @@ namespace Squidex.Domain.Apps.Entities.Backup Task DeleteAsync(Guid id); - Task ClearAsync(); - Task>> GetStateAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index fe50bca91..f26ac8cf8 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -40,6 +40,11 @@ namespace Squidex.Domain.Apps.Entities.Backup private RestoreState state = new RestoreState(); private IPersistence persistence; + private RestoreStateJob CurrentJob + { + get { return state.Job; } + } + public RestoreGrain( IAssetStore assetStore, IBackupArchiveLocation backupArchiveLocation, @@ -92,10 +97,12 @@ namespace Squidex.Domain.Apps.Entities.Backup private async Task RecoverAfterRestartAsync() { - if (state.Job?.Status == JobStatus.Started) + if (CurrentJob?.Status == JobStatus.Started) { Log("Failed due application restart"); + CurrentJob.Status = JobStatus.Failed; + await CleanupAsync(); await WriteAsync(); } @@ -105,7 +112,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { Guard.NotNull(url, nameof(url)); - if (state.Job?.Status == JobStatus.Started) + if (CurrentJob?.Status == JobStatus.Started) { throw new DomainException("A restore operation is already running."); } @@ -134,17 +141,24 @@ namespace Squidex.Domain.Apps.Entities.Backup { try { + Log("Started. The restore process has the following steps:"); + Log(" * Download backup"); + Log(" * Restore events and attachments."); + Log(" * Restore all objects like app, schemas and contents"); + Log(" * Complete the restore operation for all objects"); + log.LogInformation(w => w .WriteProperty("action", "restore") .WriteProperty("status", "started") - .WriteProperty("url", state.Job.Uri.ToString())); + .WriteProperty("operationId", CurrentJob.Id.ToString()) + .WriteProperty("url", CurrentJob.Uri.ToString())); using (Profiler.Trace("Download")) { await DownloadAsync(); } - using (var reader = await backupArchiveLocation.OpenArchiveAsync(state.Job.Id)) + using (var reader = await backupArchiveLocation.OpenArchiveAsync(CurrentJob.Id)) { using (Profiler.Trace("ReadEvents")) { @@ -155,7 +169,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RestoreAsync))) { - await handler.RestoreAsync(state.Job.AppId, reader); + await handler.RestoreAsync(CurrentJob.AppId, reader); } Log($"Restored {handler.Name}"); @@ -165,20 +179,23 @@ namespace Squidex.Domain.Apps.Entities.Backup { using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.CompleteRestoreAsync))) { - await handler.CompleteRestoreAsync(state.Job.AppId, reader); + await handler.CompleteRestoreAsync(CurrentJob.AppId, reader); } Log($"Completed {handler.Name}"); } } - state.Job.Status = JobStatus.Failed; + CurrentJob.Status = JobStatus.Completed; + + Log("Completed, Yeah!"); log.LogInformation(w => { w.WriteProperty("action", "restore"); w.WriteProperty("status", "completed"); - w.WriteProperty("url", state.Job.Uri.ToString()); + w.WriteProperty("operationId", CurrentJob.Id.ToString()); + w.WriteProperty("url", CurrentJob.Uri.ToString()); Profiler.Session?.Write(w); }); @@ -194,28 +211,24 @@ namespace Squidex.Domain.Apps.Entities.Backup Log("Failed with internal error"); } - try - { - await CleanupAsync(ex); - } - catch (Exception ex2) - { - ex = ex2; - } + await CleanupAsync(ex); - state.Job.Status = JobStatus.Failed; + CurrentJob.Status = JobStatus.Failed; log.LogError(ex, w => { w.WriteProperty("action", "retore"); w.WriteProperty("status", "failed"); - w.WriteProperty("url", state.Job.Uri.ToString()); + w.WriteProperty("operationId", CurrentJob.Id.ToString()); + w.WriteProperty("url", CurrentJob.Uri.ToString()); Profiler.Session?.Write(w); }); } finally { + CurrentJob.Stopped = clock.GetCurrentInstant(); + await WriteAsync(); } } @@ -223,16 +236,16 @@ namespace Squidex.Domain.Apps.Entities.Backup private async Task CleanupAsync(Exception exception = null) { - await backupArchiveLocation.DeleteArchiveAsync(state.Job.Id); + await Safe.DeleteAsync(backupArchiveLocation, CurrentJob.Id, log); - if (state.Job.AppId != Guid.Empty) + if (CurrentJob.AppId != Guid.Empty) { foreach (var handler in handlers) { - await handler.CleanupRestoreAsync(state.Job.AppId, exception); + await Safe.CleanupRestoreAsync(handler, CurrentJob.AppId, CurrentJob.Id, log); } - await appCleaner.EnqueueAppAsync(state.Job.AppId); + await appCleaner.EnqueueAppAsync(CurrentJob.AppId); } } @@ -240,7 +253,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { Log("Downloading Backup"); - await backupArchiveLocation.DownloadAsync(state.Job.Uri, state.Job.Id); + await backupArchiveLocation.DownloadAsync(CurrentJob.Uri, CurrentJob.Id); Log("Downloaded Backup"); } @@ -255,21 +268,22 @@ namespace Squidex.Domain.Apps.Entities.Backup { squidexEvent.Actor = actor; } - else if (@event.Payload is AppCreated appCreated) + + if (@event.Payload is AppCreated appCreated) { - state.Job.AppId = appCreated.AppId.Id; + CurrentJob.AppId = appCreated.AppId.Id; await CheckCleanupStatus(); } foreach (var handler in handlers) { - await handler.RestoreEventAsync(@event, state.Job.AppId, reader); + await handler.RestoreEventAsync(@event, CurrentJob.AppId, reader); } await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, new List { storedEvent.Data }); - Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments."); + Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments.", true); }); Log("Reading events completed."); @@ -279,7 +293,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { var cleaner = grainFactory.GetGrain(SingleGrain.Id); - var status = await cleaner.GetStatusAsync(state.Job.AppId); + var status = await cleaner.GetStatusAsync(CurrentJob.AppId); if (status == CleanerStatus.Cleaning) { @@ -292,9 +306,16 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - private void Log(string message) + private void Log(string message, bool replace = false) { - state.Job.Log.Add($"{clock.GetCurrentInstant()}: {message}"); + if (replace && CurrentJob.Log.Count > 0) + { + CurrentJob.Log[CurrentJob.Log.Count - 1] = $"{clock.GetCurrentInstant()}: {message}"; + } + else + { + CurrentJob.Log.Add($"{clock.GetCurrentInstant()}: {message}"); + } } private async Task ReadAsync() @@ -309,7 +330,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public Task> GetJobAsync() { - return Task.FromResult>(state.Job); + return Task.FromResult>(CurrentJob); } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs index 1eac3612b..6ffdf675e 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs @@ -30,7 +30,7 @@ namespace Squidex.Domain.Apps.Entities.Backup.State public Instant? Stopped { get; set; } [JsonProperty] - public List Log { get; set; } + public List Log { get; set; } = new List(); [JsonProperty] public JobStatus Status { get; set; } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs index 2ae90e535..6aa35f32e 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs @@ -24,6 +24,8 @@ namespace Squidex.Domain.Apps.Entities.Contents private readonly HashSet contentIds = new HashSet(); private readonly IContentRepository contentRepository; + public override string Name { get; } = "Contents"; + public BackupContents(IStore store, IContentRepository contentRepository) : base(store) { diff --git a/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs b/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs index 29b7bafc5..73eba9be7 100644 --- a/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs +++ b/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs @@ -17,6 +17,8 @@ namespace Squidex.Domain.Apps.Entities.History { private readonly IHistoryEventRepository historyEventRepository; + public override string Name { get; } = "History"; + public BackupHistory(IHistoryEventRepository historyEventRepository) { Guard.NotNull(historyEventRepository, nameof(historyEventRepository)); diff --git a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs index 0ea1c005d..87e6af41b 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs @@ -25,6 +25,8 @@ namespace Squidex.Domain.Apps.Entities.Rules private readonly HashSet ruleIds = new HashSet(); private readonly IGrainFactory grainFactory; + public override string Name { get; } = "Rules"; + public BackupRules(IStore store, IGrainFactory grainFactory) : base(store) { diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs index 16d68c5db..6415863b8 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs @@ -29,6 +29,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas private readonly FieldRegistry fieldRegistry; private readonly IGrainFactory grainFactory; + public override string Name { get; } = "Schemas"; + public BackupSchemas(IStore store, FieldRegistry fieldRegistry, IGrainFactory grainFactory) : base(store) { diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/BackupJobDto.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/BackupJobDto.cs index 45d6fbed8..475f4b059 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/Models/BackupJobDto.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/BackupJobDto.cs @@ -40,9 +40,9 @@ namespace Squidex.Areas.Api.Controllers.Backups.Models public int HandledAssets { get; set; } /// - /// Indicates if the job has failed. + /// The status of the operation. /// - public bool IsFailed { get; set; } + public JobStatus Status { get; set; } public static BackupJobDto FromBackup(IBackupJob backup) { diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs index a419a5f33..b0270f449 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs @@ -6,6 +6,7 @@ // ========================================================================== using System; +using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using NodaTime; using Squidex.Domain.Apps.Entities.Backup; @@ -22,20 +23,25 @@ namespace Squidex.Areas.Api.Controllers.Backups.Models public Uri Uri { get; set; } /// - /// The status text. + /// The status log. /// [Required] - public string Status { get; set; } + public List Log { get; set; } /// - /// Indicates when the restore operation has been started. + /// The time when the job has been started. /// public Instant Started { get; set; } /// - /// Indicates if the restore has failed. + /// The time when the job has been stopped. /// - public bool IsFailed { get; set; } + public Instant? Stopped { get; set; } + + /// + /// The status of the operation. + /// + public JobStatus Status { get; set; } public static RestoreJobDto FromJob(IRestoreJob job) { diff --git a/src/Squidex/app/features/apps/declarations.ts b/src/Squidex/app/features/apps/declarations.ts index 880671f15..8947fb5e9 100644 --- a/src/Squidex/app/features/apps/declarations.ts +++ b/src/Squidex/app/features/apps/declarations.ts @@ -6,4 +6,5 @@ */ export * from './pages/apps-page.component'; -export * from './pages/onboarding-dialog.component'; \ No newline at end of file +export * from './pages/onboarding-dialog.component'; +export * from './pages/restore-page.component'; \ No newline at end of file diff --git a/src/Squidex/app/features/apps/module.ts b/src/Squidex/app/features/apps/module.ts index 69c393429..51c4c718f 100644 --- a/src/Squidex/app/features/apps/module.ts +++ b/src/Squidex/app/features/apps/module.ts @@ -12,13 +12,17 @@ import { SqxFrameworkModule, SqxSharedModule } from '@app/shared'; import { AppsPageComponent, - OnboardingDialogComponent + OnboardingDialogComponent, + RestorePageComponent } from './declarations'; const routes: Routes = [ { path: '', component: AppsPageComponent + }, { + path: 'restore', + component: RestorePageComponent } ]; @@ -30,7 +34,8 @@ const routes: Routes = [ ], declarations: [ AppsPageComponent, - OnboardingDialogComponent + OnboardingDialogComponent, + RestorePageComponent ] }) export class SqxFeatureAppsModule { } \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.html b/src/Squidex/app/features/apps/pages/apps-page.component.html index 1d29c5506..b3aaf4bbe 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.html +++ b/src/Squidex/app/features/apps/pages/apps-page.component.html @@ -1,6 +1,6 @@  -
+

Hi {{authState.user?.displayName}}

@@ -8,118 +8,78 @@
-
-
-
- -
-
-

You are not collaborating to any app yet

-
- -
-
-

{{app.name}}

- -
- Edit -
-
-
-
- - -
-
-
-
- -
- -

New App

- -
- Create a new blank app without content and schemas. -
-
-
- -
-
-
- -
- -

New Blog Sample

- -
-
Start with our ready to use blog.
-
- Sample Code: C# -
-
-
-
- -
-
-
- -
- -

New Profile Sample

- -
-
Create your profile page.
-
- Sample Code: C# -
-
-
+ +
+
+

You are not collaborating to any app yet

+
+ +
+
+

{{app.name}}

+ +
+ Edit
-
-

Restore Backup

- - -
-
-
-
-
- -
-
- -
-
- -
-

Restore

-
-
-
-
- Status: {{job.status}} -
+ + +
+
+
+
+ +
+ +

New App

+ +
+ Create a new blank app without content and schemas. +
+
+
+ +
+
+
+
- -
-
-
- +

New Blog Sample

+ +
+
Start with our ready to use blog.
+
+ Sample Code: C#
-
- +
+
+
+ +
+
+
+ +
+ +

New Profile Sample

+ +
+
Create your profile page.
+
+ Sample Code: C#
- +
+
+ Restore app from backup +
+ diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.scss b/src/Squidex/app/features/apps/pages/apps-page.component.scss index 835a60c5a..af4c66b32 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.scss +++ b/src/Squidex/app/features/apps/pages/apps-page.component.scss @@ -12,20 +12,13 @@ padding-top: 2rem; padding-right: 1.25rem; padding-bottom: 0; + padding-left: $size-sidebar-width + .25rem; display: block; } } -.col-left { - padding-left: $size-sidebar-width + .25rem; -} - -.col-right { - border-left: 1px solid darken($color-border, 3%); - min-height: 200px; - min-width: 300px; - max-width: 300px; - padding: 0 2rem 0 1rem; +.restore-link { + font-size: 0.8rem; } .card { @@ -90,37 +83,4 @@ text-decoration: none; } } -} - -$cicle-size: 1.5rem; - -.restore { - &-card { - float: none; - } - - &-status { - & { - @include circle($cicle-size); - line-height: $cicle-size + .1rem; - text-align: center; - font-size: .6 * $cicle-size; - font-weight: normal; - background: $color-border; - color: $color-dark-foreground; - vertical-align: middle; - } - - &-pending { - color: inherit; - } - - &-failed { - background: $color-theme-error; - } - - &-success { - background: $color-theme-green; - } - } } \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.ts b/src/Squidex/app/features/apps/pages/apps-page.component.ts index 1414375b7..c081ccb84 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.ts +++ b/src/Squidex/app/features/apps/pages/apps-page.component.ts @@ -5,21 +5,15 @@ * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. */ -import { Component, OnDestroy, OnInit } from '@angular/core'; -import { FormBuilder } from '@angular/forms'; -import { Subscription, timer } from 'rxjs'; -import { switchMap, take } from 'rxjs/operators'; +import { Component, OnInit } from '@angular/core'; +import { take } from 'rxjs/operators'; import { AppsState, AuthService, - BackupsService, DialogModel, - DialogService, ModalModel, - OnboardingService, - RestoreDto, - RestoreForm + OnboardingService } from '@app/shared'; @Component({ @@ -27,38 +21,20 @@ import { styleUrls: ['./apps-page.component.scss'], templateUrl: './apps-page.component.html' }) -export class AppsPageComponent implements OnDestroy, OnInit { - private timerSubscription: Subscription; - +export class AppsPageComponent implements OnInit { public addAppDialog = new DialogModel(); public addAppTemplate = ''; - public restoreJob: RestoreDto | null; - public restoreForm = new RestoreForm(this.formBuilder); - public onboardingModal = new ModalModel(); constructor( public readonly appsState: AppsState, public readonly authState: AuthService, - private readonly backupsService: BackupsService, - private readonly dialogs: DialogService, - private readonly formBuilder: FormBuilder, private readonly onboardingService: OnboardingService ) { } - public ngOnDestroy() { - this.timerSubscription.unsubscribe(); - } - public ngOnInit() { - this.timerSubscription = - timer(0, 3000).pipe(switchMap(t => this.backupsService.getRestore())) - .subscribe(dto => { - this.restoreJob = dto; - }); - this.appsState.apps.pipe( take(1)) .subscribe(apps => { @@ -69,21 +45,6 @@ export class AppsPageComponent implements OnDestroy, OnInit { }); } - public restore() { - const value = this.restoreForm.submit(); - - if (value) { - this.restoreForm.submitCompleted({}); - - this.backupsService.postRestore(value.url) - .subscribe(() => { - this.dialogs.notifyInfo('Restore started, it can take several minutes to complete.'); - }, error => { - this.dialogs.notifyError(error); - }); - } - } - public createNewApp(template: string) { this.addAppTemplate = template; this.addAppDialog.show(); diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.html b/src/Squidex/app/features/apps/pages/restore-page.component.html new file mode 100644 index 000000000..775d168e1 --- /dev/null +++ b/src/Squidex/app/features/apps/pages/restore-page.component.html @@ -0,0 +1,55 @@ + + +
+ +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+

Last Restore Operation

+
+
+
+
+
+ {{row}} +
+
+
+
+
+ Started: {{job.started | sqxISODate}} +
+
+ Stopped: {{job.stopped | sqxISODate}} +
+
+
+
+ + +
+
+
+
+ +
+
+ +
+
+
+
+
\ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.scss b/src/Squidex/app/features/apps/pages/restore-page.component.scss new file mode 100644 index 000000000..5b4a356d0 --- /dev/null +++ b/src/Squidex/app/features/apps/pages/restore-page.component.scss @@ -0,0 +1,59 @@ +@import '_vars'; +@import '_mixins'; + +$circle-size: 2rem; + +h3 { + margin: 0; +} + +.section { + margin-bottom: .8rem; +} + +.container { + padding-top: 2rem; +} + +.card { + &-header { + h3 { + line-height: $circle-size; + } + } + + &-body { + font-family: monospace; + background: $color-border; + max-height: 400px; + min-height: 300px; + overflow-y: scroll; + } +} + +.restore { + &-status { + & { + @include circle($circle-size); + line-height: $circle-size + .1rem; + text-align: center; + font-size: .6 * $circle-size; + font-weight: normal; + background: $color-border; + color: $color-dark-foreground; + vertical-align: middle; + } + + &-pending { + color: inherit; + } + + &-failed { + background: $color-theme-error; + } + + &-success { + background: $color-theme-green; + } + } +} \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.ts b/src/Squidex/app/features/apps/pages/restore-page.component.ts new file mode 100644 index 000000000..cbb40bfd8 --- /dev/null +++ b/src/Squidex/app/features/apps/pages/restore-page.component.ts @@ -0,0 +1,66 @@ +/* + * Squidex Headless CMS + * + * @license + * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. + */ + +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { FormBuilder } from '@angular/forms'; +import { Subscription, timer } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; + +import { + AuthService, + BackupsService, + DialogService, + RestoreDto, + RestoreForm +} from '@app/shared'; + +@Component({ + selector: 'sqx-restore-page', + styleUrls: ['./restore-page.component.scss'], + templateUrl: './restore-page.component.html' +}) +export class RestorePageComponent implements OnDestroy, OnInit { + private timerSubscription: Subscription; + + public restoreJob: RestoreDto | null; + public restoreForm = new RestoreForm(this.formBuilder); + + constructor( + public readonly authState: AuthService, + private readonly backupsService: BackupsService, + private readonly dialogs: DialogService, + private readonly formBuilder: FormBuilder + ) { + } + + public ngOnDestroy() { + this.timerSubscription.unsubscribe(); + } + + public ngOnInit() { + this.timerSubscription = + timer(0, 1000).pipe(switchMap(() => this.backupsService.getRestore())) + .subscribe(dto => { + this.restoreJob = dto; + }); + } + + public restore() { + const value = this.restoreForm.submit(); + + if (value) { + this.restoreForm.submitCompleted({}); + + this.backupsService.postRestore(value.url) + .subscribe(() => { + this.dialogs.notifyInfo('Restore started, it can take several minutes to complete.'); + }, error => { + this.dialogs.notifyError(error); + }); + } + } +} \ No newline at end of file diff --git a/src/Squidex/app/features/rules/pages/events/rule-events-page.component.scss b/src/Squidex/app/features/rules/pages/events/rule-events-page.component.scss index db9f3550d..a05c75506 100644 --- a/src/Squidex/app/features/rules/pages/events/rule-events-page.component.scss +++ b/src/Squidex/app/features/rules/pages/events/rule-events-page.component.scss @@ -18,6 +18,7 @@ h3 { &-dump { margin-top: 1rem; + font-family: monospace; font-size: .8rem; font-weight: normal; height: 20rem; diff --git a/src/Squidex/app/features/settings/pages/backups/backups-page.component.html b/src/Squidex/app/features/settings/pages/backups/backups-page.component.html index 17f3bf72b..9c2b7aeee 100644 --- a/src/Squidex/app/features/settings/pages/backups/backups-page.component.html +++ b/src/Squidex/app/features/settings/pages/backups/backups-page.component.html @@ -35,13 +35,13 @@
-
+
-
+
-
+
diff --git a/src/Squidex/app/shared/services/backups.service.spec.ts b/src/Squidex/app/shared/services/backups.service.spec.ts index 77e24c9aa..d61640fa2 100644 --- a/src/Squidex/app/shared/services/backups.service.spec.ts +++ b/src/Squidex/app/shared/services/backups.service.spec.ts @@ -55,7 +55,7 @@ describe('BackupsService', () => { stopped: '2017-02-04', handledEvents: 13, handledAssets: 17, - isFailed: false + status: 'Failed' }, { id: '2', @@ -63,14 +63,14 @@ describe('BackupsService', () => { stopped: null, handledEvents: 23, handledAssets: 27, - isFailed: true + status: 'Completed' } ]); expect(backups!).toEqual( [ - new BackupDto('1', DateTime.parseISO_UTC('2017-02-03'), DateTime.parseISO_UTC('2017-02-04'), 13, 17, false), - new BackupDto('2', DateTime.parseISO_UTC('2018-02-03'), null, 23, 27, true) + new BackupDto('1', DateTime.parseISO_UTC('2017-02-03'), DateTime.parseISO_UTC('2017-02-04'), 13, 17, 'Failed'), + new BackupDto('2', DateTime.parseISO_UTC('2018-02-03'), null, 23, 27, 'Completed') ]); })); diff --git a/src/Squidex/app/shared/services/backups.service.ts b/src/Squidex/app/shared/services/backups.service.ts index 563b809e7..2c8bc79ec 100644 --- a/src/Squidex/app/shared/services/backups.service.ts +++ b/src/Squidex/app/shared/services/backups.service.ts @@ -26,7 +26,7 @@ export class BackupDto extends Model { public readonly stopped: DateTime | null, public readonly handledEvents: number, public readonly handledAssets: number, - public readonly isFailed: boolean + public readonly status: string ) { super(); } @@ -38,10 +38,11 @@ export class BackupDto extends Model { export class RestoreDto { constructor( + public readonly url: string, public readonly started: DateTime, + public readonly stopped: DateTime | null, public readonly status: string, - public readonly url: string, - public readonly isFailed: boolean + public readonly log: string[] ) { } } @@ -69,7 +70,7 @@ export class BackupsService { item.stopped ? DateTime.parseISO_UTC(item.stopped) : null, item.handledEvents, item.handledAssets, - item.isFailed); + item.status); }); }), pretifyError('Failed to load backups.')); @@ -83,10 +84,11 @@ export class BackupsService { const body: any = response; return new RestoreDto( + body.id, DateTime.parseISO_UTC(body.started), + body.stopped ? DateTime.parseISO_UTC(body.stopped) : null, body.status, - body.url, - body.isFailed); + body.log); }), catchError(error => { if (Types.is(error, HttpErrorResponse) && error.status === 404) { diff --git a/src/Squidex/app/shared/state/backups.state.spec.ts b/src/Squidex/app/shared/state/backups.state.spec.ts index d46cea933..ad1b63dfb 100644 --- a/src/Squidex/app/shared/state/backups.state.spec.ts +++ b/src/Squidex/app/shared/state/backups.state.spec.ts @@ -22,8 +22,8 @@ describe('BackupsState', () => { const app = 'my-app'; const oldBackups = [ - new BackupDto('id1', DateTime.now(), null, 1, 1, false), - new BackupDto('id2', DateTime.now(), null, 2, 2, false) + new BackupDto('id1', DateTime.now(), null, 1, 1, 'Started'), + new BackupDto('id2', DateTime.now(), null, 2, 2, 'Started') ]; let dialogs: IMock; From 9f93c08b21ff64a37b1a964cdca8ef02bd23d587 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 31 Jul 2018 14:43:07 +0200 Subject: [PATCH 15/23] Reduces TS size with tslib and added tests backup. --- .../administration/services/users.service.ts | 4 - .../shared/services/app-clients.service.ts | 4 - .../services/app-contributors.service.ts | 8 -- .../shared/services/app-languages.service.ts | 8 -- .../shared/services/app-patterns.service.ts | 8 -- .../app/shared/services/apps.service.ts | 4 - .../app/shared/services/assets.service.ts | 4 - .../shared/services/backups.service.spec.ts | 94 ++++++++++++++++++- .../app/shared/services/backups.service.ts | 6 +- .../shared/services/contents.service.spec.ts | 13 +++ .../app/shared/services/contents.service.ts | 6 +- .../app/shared/services/plans.service.ts | 8 -- .../app/shared/services/rules.service.spec.ts | 5 + .../app/shared/services/rules.service.ts | 8 -- src/Squidex/package.json | 1 + src/Squidex/tsconfig.json | 2 + 16 files changed, 116 insertions(+), 67 deletions(-) diff --git a/src/Squidex/app/features/administration/services/users.service.ts b/src/Squidex/app/features/administration/services/users.service.ts index 4b23dbff4..8ffa5cabb 100644 --- a/src/Squidex/app/features/administration/services/users.service.ts +++ b/src/Squidex/app/features/administration/services/users.service.ts @@ -24,10 +24,6 @@ export class UsersDto extends Model { ) { super(); } - - public with(value: Partial): UsersDto { - return this.clone(value); - } } export class UserDto extends Model { diff --git a/src/Squidex/app/shared/services/app-clients.service.ts b/src/Squidex/app/shared/services/app-clients.service.ts index 2d4186ecd..81aede153 100644 --- a/src/Squidex/app/shared/services/app-clients.service.ts +++ b/src/Squidex/app/shared/services/app-clients.service.ts @@ -27,10 +27,6 @@ export class AppClientsDto extends Model { ) { super(); } - - public with(value: Partial): AppClientsDto { - return this.clone(value); - } } export class AppClientDto extends Model { diff --git a/src/Squidex/app/shared/services/app-contributors.service.ts b/src/Squidex/app/shared/services/app-contributors.service.ts index 9b90e4ad9..bad2c9770 100644 --- a/src/Squidex/app/shared/services/app-contributors.service.ts +++ b/src/Squidex/app/shared/services/app-contributors.service.ts @@ -28,10 +28,6 @@ export class AppContributorsDto extends Model { ) { super(); } - - public with(value: Partial): AppContributorsDto { - return this.clone(value); - } } export class AppContributorDto extends Model { @@ -41,10 +37,6 @@ export class AppContributorDto extends Model { ) { super(); } - - public with(value: Partial): AppContributorDto { - return this.clone(value); - } } export class ContributorAssignedDto { diff --git a/src/Squidex/app/shared/services/app-languages.service.ts b/src/Squidex/app/shared/services/app-languages.service.ts index 936c30f04..7f68eaa6a 100644 --- a/src/Squidex/app/shared/services/app-languages.service.ts +++ b/src/Squidex/app/shared/services/app-languages.service.ts @@ -27,10 +27,6 @@ export class AppLanguagesDto extends Model { ) { super(); } - - public with(value: Partial): AppLanguagesDto { - return this.clone(value); - } } export class AppLanguageDto extends Model { @@ -43,10 +39,6 @@ export class AppLanguageDto extends Model { ) { super(); } - - public with(value: Partial): AppLanguageDto { - return this.clone(value); - } } export class AddAppLanguageDto { diff --git a/src/Squidex/app/shared/services/app-patterns.service.ts b/src/Squidex/app/shared/services/app-patterns.service.ts index b32979415..31f688a22 100644 --- a/src/Squidex/app/shared/services/app-patterns.service.ts +++ b/src/Squidex/app/shared/services/app-patterns.service.ts @@ -26,10 +26,6 @@ export class AppPatternsDto extends Model { ) { super(); } - - public with(value: Partial): AppPatternsDto { - return this.clone(value); - } } export class AppPatternDto extends Model { @@ -41,10 +37,6 @@ export class AppPatternDto extends Model { ) { super(); } - - public with(value: Partial): AppPatternDto { - return this.clone(value); - } } export class EditAppPatternDto { diff --git a/src/Squidex/app/shared/services/apps.service.ts b/src/Squidex/app/shared/services/apps.service.ts index 59d647ac4..792c8575d 100644 --- a/src/Squidex/app/shared/services/apps.service.ts +++ b/src/Squidex/app/shared/services/apps.service.ts @@ -31,10 +31,6 @@ export class AppDto extends Model { ) { super(); } - - public with(value: Partial): AppDto { - return this.clone(value); - } } export class CreateAppDto { diff --git a/src/Squidex/app/shared/services/assets.service.ts b/src/Squidex/app/shared/services/assets.service.ts index 0f781729b..e2436b4f7 100644 --- a/src/Squidex/app/shared/services/assets.service.ts +++ b/src/Squidex/app/shared/services/assets.service.ts @@ -29,10 +29,6 @@ export class AssetsDto extends Model { ) { super(); } - - public with(value: Partial): AssetsDto { - return this.clone(value); - } } export class AssetDto extends Model { diff --git a/src/Squidex/app/shared/services/backups.service.spec.ts b/src/Squidex/app/shared/services/backups.service.spec.ts index d61640fa2..22a51cb2d 100644 --- a/src/Squidex/app/shared/services/backups.service.spec.ts +++ b/src/Squidex/app/shared/services/backups.service.spec.ts @@ -13,7 +13,8 @@ import { ApiUrlConfig, BackupDto, BackupsService, - DateTime + DateTime, + RestoreDto } from './../'; describe('BackupsService', () => { @@ -74,6 +75,84 @@ describe('BackupsService', () => { ]); })); + it('should make get request to get restore', + inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { + + let restore: RestoreDto; + + backupsService.getRestore().subscribe(result => { + restore = result!; + }); + + const req = httpMock.expectOne('http://service/p/api/apps/restore'); + + expect(req.request.method).toEqual('GET'); + expect(req.request.headers.get('If-Match')).toBeNull(); + + req.flush({ + url: 'http://url', + started: '2017-02-03', + stopped: '2017-02-04', + status: 'Failed', + log: [ + 'log1', + 'log2' + ] + }); + + expect(restore!).toEqual( + new RestoreDto('http://url', + DateTime.parseISO_UTC('2017-02-03'), + DateTime.parseISO_UTC('2017-02-04'), + 'Failed', + [ + 'log1', + 'log2' + ])); + })); + + it('should return null when get restore return 404', + inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { + + let restore: RestoreDto | null; + + backupsService.getRestore().subscribe(result => { + restore = result; + }); + + const req = httpMock.expectOne('http://service/p/api/apps/restore'); + + expect(req.request.method).toEqual('GET'); + expect(req.request.headers.get('If-Match')).toBeNull(); + + req.flush({}, { status: 404, statusText: '404' }); + + expect(restore!).toBeNull(); + })); + + it('should throw error when get restore return non 404', + inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { + + let restore: RestoreDto | null; + let error: any; + + backupsService.getRestore().subscribe(result => { + restore = result; + }, err => { + error = err; + }); + + const req = httpMock.expectOne('http://service/p/api/apps/restore'); + + expect(req.request.method).toEqual('GET'); + expect(req.request.headers.get('If-Match')).toBeNull(); + + req.flush({}, { status: 500, statusText: '500' }); + + expect(restore!).toBeUndefined(); + expect(error)!.toBeDefined(); + })); + it('should make post request to start backup', inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { @@ -87,6 +166,19 @@ describe('BackupsService', () => { req.flush({}); })); + it('should make post request to start restore', + inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { + + backupsService.postRestore('http://url').subscribe(); + + const req = httpMock.expectOne('http://service/p/api/apps/restore'); + + expect(req.request.method).toEqual('POST'); + expect(req.request.headers.get('If-Match')).toBeNull(); + + req.flush({}); + })); + it('should make delete request to remove language', inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { diff --git a/src/Squidex/app/shared/services/backups.service.ts b/src/Squidex/app/shared/services/backups.service.ts index 2c8bc79ec..676e968a9 100644 --- a/src/Squidex/app/shared/services/backups.service.ts +++ b/src/Squidex/app/shared/services/backups.service.ts @@ -30,10 +30,6 @@ export class BackupDto extends Model { ) { super(); } - - public with(value: Partial): BackupDto { - return this.clone(value); - } } export class RestoreDto { @@ -84,7 +80,7 @@ export class BackupsService { const body: any = response; return new RestoreDto( - body.id, + body.url, DateTime.parseISO_UTC(body.started), body.stopped ? DateTime.parseISO_UTC(body.stopped) : null, body.status, diff --git a/src/Squidex/app/shared/services/contents.service.spec.ts b/src/Squidex/app/shared/services/contents.service.spec.ts index 8d92f9763..6e9d7fb9f 100644 --- a/src/Squidex/app/shared/services/contents.service.spec.ts +++ b/src/Squidex/app/shared/services/contents.service.spec.ts @@ -285,6 +285,19 @@ describe('ContentsService', () => { req.flush({}); })); + it('should make put request to discard changes', + inject([ContentsService, HttpTestingController], (contentsService: ContentsService, httpMock: HttpTestingController) => { + + contentsService.discardChanges('my-app', 'my-schema', 'content1', version).subscribe(); + + const req = httpMock.expectOne('http://service/p/api/content/my-app/my-schema/content1/discard'); + + expect(req.request.method).toEqual('PUT'); + expect(req.request.headers.get('If-Match')).toBe(version.value); + + req.flush({}); + })); + it('should make put request to change content status', inject([ContentsService, HttpTestingController], (contentsService: ContentsService, httpMock: HttpTestingController) => { diff --git a/src/Squidex/app/shared/services/contents.service.ts b/src/Squidex/app/shared/services/contents.service.ts index d1a2eed27..9052d8852 100644 --- a/src/Squidex/app/shared/services/contents.service.ts +++ b/src/Squidex/app/shared/services/contents.service.ts @@ -43,10 +43,6 @@ export class ScheduleDto extends Model { ) { super(); } - - public with(value: Partial): ScheduleDto { - return this.clone(value); - } } export class ContentDto extends Model { @@ -233,7 +229,7 @@ export class ContentsService { public discardChanges(appName: string, schemaName: string, id: string, version: Version): Observable> { const url = this.apiUrl.buildUrl(`/api/content/${appName}/${schemaName}/${id}/discard`); - return HTTP.putVersioned(this.http, url, version).pipe( + return HTTP.putVersioned(this.http, url, {}, version).pipe( tap(() => { this.analytics.trackEvent('Content', 'Discarded', appName); }), diff --git a/src/Squidex/app/shared/services/plans.service.ts b/src/Squidex/app/shared/services/plans.service.ts index 491a6062c..9ab81561f 100644 --- a/src/Squidex/app/shared/services/plans.service.ts +++ b/src/Squidex/app/shared/services/plans.service.ts @@ -30,10 +30,6 @@ export class PlansDto extends Model { ) { super(); } - - public with(value: Partial): PlansDto { - return this.clone(value); - } } export class PlanDto extends Model { @@ -49,10 +45,6 @@ export class PlanDto extends Model { ) { super(); } - - public with(value: Partial): PlanDto { - return this.clone(value); - } } export class PlanChangedDto { diff --git a/src/Squidex/app/shared/services/rules.service.spec.ts b/src/Squidex/app/shared/services/rules.service.spec.ts index 3905ffa67..3d6927f41 100644 --- a/src/Squidex/app/shared/services/rules.service.spec.ts +++ b/src/Squidex/app/shared/services/rules.service.spec.ts @@ -201,6 +201,8 @@ describe('RulesService', () => { expect(req.request.method).toEqual('DELETE'); expect(req.request.headers.get('If-Match')).toEqual(version.value); + + req.flush({}); })); it('should make get request to get app rule events', @@ -265,5 +267,8 @@ describe('RulesService', () => { const req = httpMock.expectOne('http://service/p/api/apps/my-app/rules/events/123'); expect(req.request.method).toEqual('PUT'); + expect(req.request.headers.get('If-Match')).toBeNull(); + + req.flush({}); })); }); \ No newline at end of file diff --git a/src/Squidex/app/shared/services/rules.service.ts b/src/Squidex/app/shared/services/rules.service.ts index e5df0d780..fd4b79fb0 100644 --- a/src/Squidex/app/shared/services/rules.service.ts +++ b/src/Squidex/app/shared/services/rules.service.ts @@ -83,10 +83,6 @@ export class RuleEventsDto extends Model { ) { super(); } - - public with(value: Partial): RuleEventsDto { - return this.clone(value); - } } export class RuleEventDto extends Model { @@ -103,10 +99,6 @@ export class RuleEventDto extends Model { ) { super(); } - - public with(value: Partial): RuleEventDto { - return this.clone(value); - } } export class CreateRuleDto { diff --git a/src/Squidex/package.json b/src/Squidex/package.json index 5ab170f23..3bc7cc1c6 100644 --- a/src/Squidex/package.json +++ b/src/Squidex/package.json @@ -43,6 +43,7 @@ "rxjs": "6.2.0", "slugify": "1.3.0", "sortablejs": "1.7.0", + "tslib": "^1.9.3", "zone.js": "0.8.26" }, "devDependencies": { diff --git a/src/Squidex/tsconfig.json b/src/Squidex/tsconfig.json index af90295f4..f93597800 100644 --- a/src/Squidex/tsconfig.json +++ b/src/Squidex/tsconfig.json @@ -3,8 +3,10 @@ "baseUrl": ".", "emitDecoratorMetadata": true, "experimentalDecorators": true, + "importHelpers": true, "lib": ["es6", "esnext", "dom"], "moduleResolution": "node", + "noEmitHelpers": true, "noImplicitAny": true, "noUnusedLocals": true, "noUnusedParameters": false, From b0f2951cac5fbd453c638acf9a6d9bea1b5d5370 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 31 Jul 2018 20:44:49 +0200 Subject: [PATCH 16/23] A lot of cleanup operations. --- .../Rules/MongoRuleEventRepository.cs | 5 ++++ .../Apps/Indexes/AppsByNameIndexGrain.cs | 13 +++++++++- .../Backup/BackupReader.cs | 2 +- .../Backup/BackupWriter.cs | 2 +- .../Backup/IRestoreJob.cs | 2 +- .../Backup/RestoreGrain.cs | 25 ++++++++++++++----- .../Backup/State/RestoreStateJob.cs | 2 +- .../Rules/BackupRules.cs | 11 +++++--- .../Repositories/IRuleEventRepository.cs | 2 ++ .../Backups/Models/RestoreJobDto.cs | 2 +- .../Controllers/Backups/RestoreController.cs | 1 - .../apps/pages/restore-page.component.html | 6 ++++- .../apps/pages/restore-page.component.scss | 8 ++++++ .../apps/pages/restore-page.component.ts | 6 +++-- 14 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Rules/MongoRuleEventRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Rules/MongoRuleEventRepository.cs index d7858ebcf..b775e9aba 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Rules/MongoRuleEventRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Rules/MongoRuleEventRepository.cs @@ -65,6 +65,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Rules return ruleEvent; } + public Task RemoveAsync(Guid appId) + { + return Collection.DeleteManyAsync(x => x.AppId == appId); + } + public async Task CountByAppAsync(Guid appId) { return (int)await Collection.CountDocumentsAsync(x => x.AppId == appId); diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs index bcf7d791b..29da56d76 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs @@ -83,12 +83,23 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes { state.Apps[name] = appId; + reservedIds.Remove(appId); + reservedNames.Remove(name); + return persistence.WriteSnapshotAsync(state); } public Task RemoveAppAsync(Guid appId) { - state.Apps.Remove(state.Apps.FirstOrDefault(x => x.Value == appId).Key ?? string.Empty); + var name = state.Apps.FirstOrDefault(x => x.Value == appId).Key; + + if (!string.IsNullOrWhiteSpace(name)) + { + state.Apps.Remove(name); + + reservedIds.Remove(appId); + reservedNames.Remove(name); + } return persistence.WriteSnapshotAsync(state); } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs index 731cc92e2..bb43a04ab 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs @@ -35,7 +35,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public BackupReader(Stream stream) { - archive = new ZipArchive(stream, ZipArchiveMode.Read, true); + archive = new ZipArchive(stream, ZipArchiveMode.Read, false); } protected override void DisposeObject(bool disposing) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs index 5f424036c..9ee875e3b 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs @@ -35,7 +35,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public BackupWriter(Stream stream) { - archive = new ZipArchive(stream, ZipArchiveMode.Update, true); + archive = new ZipArchive(stream, ZipArchiveMode.Create, false); } protected override void DisposeObject(bool disposing) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs index 8d0db185c..fdd5306d8 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs @@ -13,7 +13,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { public interface IRestoreJob { - Uri Uri { get; } + Uri Url { get; } Instant Started { get; } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index f26ac8cf8..5ba6721c3 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -122,7 +122,7 @@ namespace Squidex.Domain.Apps.Entities.Backup Id = Guid.NewGuid(), Started = clock.GetCurrentInstant(), Status = JobStatus.Started, - Uri = url + Url = url }; Process(); @@ -151,7 +151,7 @@ namespace Squidex.Domain.Apps.Entities.Backup .WriteProperty("action", "restore") .WriteProperty("status", "started") .WriteProperty("operationId", CurrentJob.Id.ToString()) - .WriteProperty("url", CurrentJob.Uri.ToString())); + .WriteProperty("url", CurrentJob.Url.ToString())); using (Profiler.Trace("Download")) { @@ -195,7 +195,7 @@ namespace Squidex.Domain.Apps.Entities.Backup w.WriteProperty("action", "restore"); w.WriteProperty("status", "completed"); w.WriteProperty("operationId", CurrentJob.Id.ToString()); - w.WriteProperty("url", CurrentJob.Uri.ToString()); + w.WriteProperty("url", CurrentJob.Url.ToString()); Profiler.Session?.Write(w); }); @@ -220,7 +220,7 @@ namespace Squidex.Domain.Apps.Entities.Backup w.WriteProperty("action", "retore"); w.WriteProperty("status", "failed"); w.WriteProperty("operationId", CurrentJob.Id.ToString()); - w.WriteProperty("url", CurrentJob.Uri.ToString()); + w.WriteProperty("url", CurrentJob.Url.ToString()); Profiler.Session?.Write(w); }); @@ -253,13 +253,15 @@ namespace Squidex.Domain.Apps.Entities.Backup { Log("Downloading Backup"); - await backupArchiveLocation.DownloadAsync(CurrentJob.Uri, CurrentJob.Id); + await backupArchiveLocation.DownloadAsync(CurrentJob.Url, CurrentJob.Id); Log("Downloaded Backup"); } private async Task ReadEventsAsync(BackupReader reader) { + var isOwnerAdded = false; + await reader.ReadEventsAsync(async (storedEvent) => { var @event = eventDataFormatter.Parse(storedEvent.Data); @@ -275,13 +277,24 @@ namespace Squidex.Domain.Apps.Entities.Backup await CheckCleanupStatus(); } + else if (@event.Payload is AppContributorAssigned appContributorAssigned) + { + if (!isOwnerAdded) + { + isOwnerAdded = true; + + appContributorAssigned.ContributorId = actor.Identifier; + } + } foreach (var handler in handlers) { await handler.RestoreEventAsync(@event, CurrentJob.AppId, reader); } - await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, new List { storedEvent.Data }); + var eventData = eventDataFormatter.ToEventData(@event, @event.Headers.CommitId()); + + await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, new List { eventData }); Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments.", true); }); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs index 6ffdf675e..4140fcae2 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs @@ -21,7 +21,7 @@ namespace Squidex.Domain.Apps.Entities.Backup.State public Guid AppId { get; set; } [JsonProperty] - public Uri Uri { get; set; } + public Uri Url { get; set; } [JsonProperty] public Instant Started { get; set; } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs index 87e6af41b..4243159a7 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Entities.Backup; using Squidex.Domain.Apps.Entities.Rules.Indexes; +using Squidex.Domain.Apps.Entities.Rules.Repositories; using Squidex.Domain.Apps.Entities.Rules.State; using Squidex.Domain.Apps.Events.Rules; using Squidex.Infrastructure; @@ -24,20 +25,24 @@ namespace Squidex.Domain.Apps.Entities.Rules { private readonly HashSet ruleIds = new HashSet(); private readonly IGrainFactory grainFactory; + private readonly IRuleEventRepository ruleEventRepository; public override string Name { get; } = "Rules"; - public BackupRules(IStore store, IGrainFactory grainFactory) + public BackupRules(IStore store, IGrainFactory grainFactory, IRuleEventRepository ruleEventRepository) : base(store) { Guard.NotNull(grainFactory, nameof(grainFactory)); + Guard.NotNull(ruleEventRepository, nameof(ruleEventRepository)); this.grainFactory = grainFactory; + + this.ruleEventRepository = ruleEventRepository; } public override async Task RemoveAsync(Guid appId) { - var index = grainFactory.GetGrain(appId); + await grainFactory.GetGrain(appId).ClearAsync(); var idsToRemove = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); @@ -46,7 +51,7 @@ namespace Squidex.Domain.Apps.Entities.Rules await RemoveSnapshotAsync(ruleId); } - await index.ClearAsync(); + await ruleEventRepository.RemoveAsync(appId); } public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Repositories/IRuleEventRepository.cs b/src/Squidex.Domain.Apps.Entities/Rules/Repositories/IRuleEventRepository.cs index 31c09131b..37b001f72 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Repositories/IRuleEventRepository.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Repositories/IRuleEventRepository.cs @@ -25,6 +25,8 @@ namespace Squidex.Domain.Apps.Entities.Rules.Repositories Task QueryPendingAsync(Instant now, Func callback, CancellationToken ct = default(CancellationToken)); + Task RemoveAsync(Guid appId); + Task CountByAppAsync(Guid appId); Task> QueryByAppAsync(Guid appId, int skip = 0, int take = 20); diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs index b0270f449..d28d77ab0 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs @@ -20,7 +20,7 @@ namespace Squidex.Areas.Api.Controllers.Backups.Models /// The uri to load from. /// [Required] - public Uri Uri { get; set; } + public Uri Url { get; set; } /// /// The status log. diff --git a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs index 8c133ff79..26e9dc123 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs @@ -13,7 +13,6 @@ using Squidex.Areas.Api.Controllers.Backups.Models; using Squidex.Domain.Apps.Entities.Backup; using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Security; -using Squidex.Infrastructure.Tasks; using Squidex.Pipeline; namespace Squidex.Areas.Api.Controllers.Backups diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.html b/src/Squidex/app/features/apps/pages/restore-page.component.html index 775d168e1..d4f481042 100644 --- a/src/Squidex/app/features/apps/pages/restore-page.component.html +++ b/src/Squidex/app/features/apps/pages/restore-page.component.html @@ -20,6 +20,10 @@

Last Restore Operation

+ +
+ {{job.url}} +
@@ -27,7 +31,7 @@ {{row}}
-
+
Started: {{job.started | sqxISODate}} diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.scss b/src/Squidex/app/features/apps/pages/restore-page.component.scss index 5b4a356d0..38c020056 100644 --- a/src/Squidex/app/features/apps/pages/restore-page.component.scss +++ b/src/Squidex/app/features/apps/pages/restore-page.component.scss @@ -22,6 +22,10 @@ h3 { } } + &-footer { + font-size: .9rem; + } + &-body { font-family: monospace; background: $color-border; @@ -56,4 +60,8 @@ h3 { background: $color-theme-green; } } + + &-url { + line-height: 30px; + } } \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.ts b/src/Squidex/app/features/apps/pages/restore-page.component.ts index cbb40bfd8..bc3883734 100644 --- a/src/Squidex/app/features/apps/pages/restore-page.component.ts +++ b/src/Squidex/app/features/apps/pages/restore-page.component.ts @@ -43,9 +43,11 @@ export class RestorePageComponent implements OnDestroy, OnInit { public ngOnInit() { this.timerSubscription = - timer(0, 1000).pipe(switchMap(() => this.backupsService.getRestore())) + timer(0, 2000).pipe(switchMap(() => this.backupsService.getRestore())) .subscribe(dto => { - this.restoreJob = dto; + if (dto !== null) { + this.restoreJob = dto; + } }); } From 4a712a8a04b27b068af8169a31d2aa1b31269978 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 31 Jul 2018 21:20:51 +0200 Subject: [PATCH 17/23] Admin feature. --- .../Apps/BackupApps.cs | 6 - .../Assets/BackupAssets.cs | 7 - .../Backup/AppCleanerGrain.cs | 213 ----------------- .../Backup/BackupHandler.cs | 5 - .../Backup/CleanerStatus.cs | 16 -- .../Backup/EnqueueAppToCleanerMiddleware.cs | 44 ---- .../Backup/IAppCleanerGrain.cs | 20 -- .../Backup/RestoreGrain.cs | 24 -- .../Contents/BackupContents.cs | 5 - .../History/BackupHistory.cs | 34 --- .../Rules/BackupRules.cs | 14 -- .../Schemas/BackupSchemas.cs | 14 -- .../Controllers/Backups/RestoreController.cs | 18 +- src/Squidex/Config/Domain/EntitiesServices.cs | 6 - src/Squidex/Config/Orleans/SiloWrapper.cs | 2 - .../administration-area.component.html | 5 + .../features/administration/declarations.ts | 1 + .../app/features/administration/module.ts | 6 + .../pages/restore/restore-page.component.html | 65 ++++++ .../restore}/restore-page.component.scss | 1 + .../pages/restore}/restore-page.component.ts | 0 src/Squidex/app/features/apps/declarations.ts | 3 +- src/Squidex/app/features/apps/module.ts | 9 +- .../apps/pages/apps-page.component.html | 4 - .../apps/pages/apps-page.component.scss | 4 - .../apps/pages/restore-page.component.html | 59 ----- src/Squidex/app/theme/icomoon/demo.html | 18 +- .../app/theme/icomoon/fonts/icomoon.eot | Bin 25124 -> 25320 bytes .../app/theme/icomoon/fonts/icomoon.svg | 1 + .../app/theme/icomoon/fonts/icomoon.ttf | Bin 24960 -> 25156 bytes .../app/theme/icomoon/fonts/icomoon.woff | Bin 25036 -> 25232 bytes src/Squidex/app/theme/icomoon/selection.json | 219 ++++++++++-------- src/Squidex/app/theme/icomoon/style.css | 13 +- .../EnqueueAppToCleanerMiddlewareTests.cs | 48 ---- 34 files changed, 233 insertions(+), 651 deletions(-) delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs create mode 100644 src/Squidex/app/features/administration/pages/restore/restore-page.component.html rename src/Squidex/app/features/{apps/pages => administration/pages/restore}/restore-page.component.scss (97%) rename src/Squidex/app/features/{apps/pages => administration/pages/restore}/restore-page.component.ts (100%) delete mode 100644 src/Squidex/app/features/apps/pages/restore-page.component.html delete mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs index 34d2c9bcd..398456c14 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -10,7 +10,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Entities.Apps.Indexes; -using Squidex.Domain.Apps.Entities.Apps.State; using Squidex.Domain.Apps.Entities.Backup; using Squidex.Domain.Apps.Events.Apps; using Squidex.Infrastructure; @@ -37,11 +36,6 @@ namespace Squidex.Domain.Apps.Entities.Apps this.grainFactory = grainFactory; } - public override Task RemoveAsync(Guid appId) - { - return RemoveSnapshotAsync(appId); - } - public async override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) diff --git a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index 3061a39af..b6d59f386 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -52,13 +52,6 @@ namespace Squidex.Domain.Apps.Entities.Assets this.tagService = tagService; } - public override async Task RemoveAsync(Guid appId) - { - await tagService.ClearAsync(appId, TagGroups.Assets); - - await assetRepository.RemoveAsync(appId); - } - public override Task BackupEventAsync(EventData @event, Guid appId, BackupWriter writer) { if (@event.Type == "AssetCreatedEvent" || diff --git a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs deleted file mode 100644 index c833350c0..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/AppCleanerGrain.cs +++ /dev/null @@ -1,213 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Orleans; -using Orleans.Concurrency; -using Orleans.Runtime; -using Squidex.Infrastructure; -using Squidex.Infrastructure.EventSourcing; -using Squidex.Infrastructure.Log; -using Squidex.Infrastructure.Orleans; -using Squidex.Infrastructure.States; -using Squidex.Infrastructure.Tasks; - -namespace Squidex.Domain.Apps.Entities.Backup -{ - [Reentrant] - public sealed class AppCleanerGrain : GrainOfString, IRemindable, IAppCleanerGrain - { - private readonly IGrainFactory grainFactory; - private readonly IStore store; - private readonly IEventStore eventStore; - private readonly IEnumerable handlers; - private readonly ISemanticLog log; - private IPersistence persistence; - private bool isCleaning; - private State state = new State(); - - [CollectionName("AppCleaner")] - public sealed class State - { - public HashSet Apps { get; set; } = new HashSet(); - - public HashSet FailedApps { get; set; } = new HashSet(); - } - - public AppCleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable handlers, ISemanticLog log) - { - Guard.NotNull(grainFactory, nameof(grainFactory)); - Guard.NotNull(store, nameof(store)); - Guard.NotNull(handlers, nameof(handlers)); - Guard.NotNull(eventStore, nameof(eventStore)); - Guard.NotNull(log, nameof(log)); - - this.grainFactory = grainFactory; - - this.store = store; - this.handlers = handlers; - - this.log = log; - - this.eventStore = eventStore; - } - - public async override Task OnActivateAsync(string key) - { - await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(1)); - - persistence = store.WithSnapshots(Guid.Empty, s => - { - state = s; - }); - - await ReadAsync(); - - Clean(); - } - - public Task EnqueueAppAsync(Guid appId) - { - if (appId != Guid.Empty) - { - state.Apps.Add(appId); - - Clean(); - } - - return WriteAsync(); - } - - public Task ActivateAsync() - { - Clean(); - - return TaskHelper.Done; - } - - public Task ReceiveReminder(string reminderName, TickStatus status) - { - Clean(); - - return TaskHelper.Done; - } - - public Task GetStatusAsync(Guid appId) - { - if (state.Apps.Contains(appId)) - { - return Task.FromResult(CleanerStatus.Cleaning); - } - - if (state.FailedApps.Contains(appId)) - { - return Task.FromResult(CleanerStatus.Failed); - } - - return Task.FromResult(CleanerStatus.Cleaned); - } - - private void Clean() - { - CleanAsync().Forget(); - } - - private async Task CleanAsync() - { - if (isCleaning) - { - return; - } - - isCleaning = true; - try - { - foreach (var appId in state.Apps.ToList()) - { - await CleanupAppAsync(appId); - } - } - finally - { - isCleaning = false; - } - } - - private async Task CleanupAppAsync(Guid appId) - { - using (Profiler.StartSession()) - { - try - { - log.LogInformation(w => w - .WriteProperty("action", "cleanApp") - .WriteProperty("status", "started") - .WriteProperty("appId", appId.ToString())); - - await CleanupAppCoreAsync(appId); - - log.LogInformation(w => - { - w.WriteProperty("action", "cleanApp"); - w.WriteProperty("status", "completed"); - w.WriteProperty("appId", appId.ToString()); - - Profiler.Session?.Write(w); - }); - } - catch (Exception ex) - { - state.FailedApps.Add(appId); - - log.LogError(ex, w => - { - w.WriteProperty("action", "cleanApp"); - w.WriteProperty("status", "failed"); - w.WriteProperty("appId", appId.ToString()); - - Profiler.Session?.Write(w); - }); - } - finally - { - state.Apps.Remove(appId); - - await WriteAsync(); - } - } - } - - private async Task CleanupAppCoreAsync(Guid appId) - { - using (Profiler.Trace("DeleteEvents")) - { - await eventStore.DeleteManyAsync("AppId", appId.ToString()); - } - - foreach (var handler in handlers) - { - using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RemoveAsync))) - { - await handler.RemoveAsync(appId); - } - } - } - - private async Task ReadAsync() - { - await persistence.ReadAsync(); - } - - private async Task WriteAsync() - { - await persistence.WriteSnapshotAsync(state); - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs index d879e8639..645792727 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs @@ -50,10 +50,5 @@ namespace Squidex.Domain.Apps.Entities.Backup { return TaskHelper.Done; } - - public virtual Task RemoveAsync(Guid appId) - { - return TaskHelper.Done; - } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs deleted file mode 100644 index 0016388f2..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/CleanerStatus.cs +++ /dev/null @@ -1,16 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -namespace Squidex.Domain.Apps.Entities.Backup -{ - public enum CleanerStatus - { - Cleaned, - Cleaning, - Failed - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs deleted file mode 100644 index 2d479892b..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/EnqueueAppToCleanerMiddleware.cs +++ /dev/null @@ -1,44 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using Orleans; -using Squidex.Domain.Apps.Entities.Apps.Commands; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Commands; -using Squidex.Infrastructure.Orleans; - -namespace Squidex.Domain.Apps.Entities.Backup -{ - public sealed class EnqueueAppToCleanerMiddleware : ICommandMiddleware - { - private readonly IAppCleanerGrain cleaner; - - public EnqueueAppToCleanerMiddleware(IGrainFactory grainFactory) - { - Guard.NotNull(grainFactory, nameof(grainFactory)); - - cleaner = grainFactory.GetGrain(SingleGrain.Id); - } - - public async Task HandleAsync(CommandContext context, Func next) - { - if (context.IsCompleted) - { - switch (context.Command) - { - case ArchiveApp archiveApp: - await cleaner.EnqueueAppAsync(archiveApp.AppId); - break; - } - } - - await next(); - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs deleted file mode 100644 index 9e35c7165..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/IAppCleanerGrain.cs +++ /dev/null @@ -1,20 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using Squidex.Infrastructure.Orleans; - -namespace Squidex.Domain.Apps.Entities.Backup -{ - public interface IAppCleanerGrain : IBackgroundGrain - { - Task EnqueueAppAsync(Guid appId); - - Task GetStatusAsync(Guid appId); - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index 5ba6721c3..61acfa026 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -30,7 +30,6 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IAssetStore assetStore; private readonly IEventDataFormatter eventDataFormatter; private readonly IGrainFactory grainFactory; - private readonly IAppCleanerGrain appCleaner; private readonly ISemanticLog log; private readonly IEventStore eventStore; private readonly IBackupArchiveLocation backupArchiveLocation; @@ -75,8 +74,6 @@ namespace Squidex.Domain.Apps.Entities.Backup this.handlers = handlers; this.store = store; this.log = log; - - appCleaner = grainFactory.GetGrain(SingleGrain.Id); } public override async Task OnActivateAsync(string key) @@ -244,8 +241,6 @@ namespace Squidex.Domain.Apps.Entities.Backup { await Safe.CleanupRestoreAsync(handler, CurrentJob.AppId, CurrentJob.Id, log); } - - await appCleaner.EnqueueAppAsync(CurrentJob.AppId); } } @@ -274,8 +269,6 @@ namespace Squidex.Domain.Apps.Entities.Backup if (@event.Payload is AppCreated appCreated) { CurrentJob.AppId = appCreated.AppId.Id; - - await CheckCleanupStatus(); } else if (@event.Payload is AppContributorAssigned appContributorAssigned) { @@ -302,23 +295,6 @@ namespace Squidex.Domain.Apps.Entities.Backup Log("Reading events completed."); } - private async Task CheckCleanupStatus() - { - var cleaner = grainFactory.GetGrain(SingleGrain.Id); - - var status = await cleaner.GetStatusAsync(CurrentJob.AppId); - - if (status == CleanerStatus.Cleaning) - { - throw new BackupRestoreException("The app is removed in the background."); - } - - if (status == CleanerStatus.Cleaning) - { - throw new BackupRestoreException("The app could not be cleaned."); - } - } - private void Log(string message, bool replace = false) { if (replace && CurrentJob.Log.Count > 0) diff --git a/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs index 6aa35f32e..40c2eb0d4 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs @@ -34,11 +34,6 @@ namespace Squidex.Domain.Apps.Entities.Contents this.contentRepository = contentRepository; } - public override Task RemoveAsync(Guid appId) - { - return contentRepository.RemoveAsync(appId); - } - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) diff --git a/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs b/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs deleted file mode 100644 index 73eba9be7..000000000 --- a/src/Squidex.Domain.Apps.Entities/History/BackupHistory.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using Squidex.Domain.Apps.Entities.Backup; -using Squidex.Domain.Apps.Entities.History.Repositories; -using Squidex.Infrastructure; - -namespace Squidex.Domain.Apps.Entities.History -{ - public sealed class BackupHistory : BackupHandler - { - private readonly IHistoryEventRepository historyEventRepository; - - public override string Name { get; } = "History"; - - public BackupHistory(IHistoryEventRepository historyEventRepository) - { - Guard.NotNull(historyEventRepository, nameof(historyEventRepository)); - - this.historyEventRepository = historyEventRepository; - } - - public override Task RemoveAsync(Guid appId) - { - return historyEventRepository.RemoveAsync(appId); - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs index 4243159a7..a6260d6c2 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs @@ -40,20 +40,6 @@ namespace Squidex.Domain.Apps.Entities.Rules this.ruleEventRepository = ruleEventRepository; } - public override async Task RemoveAsync(Guid appId) - { - await grainFactory.GetGrain(appId).ClearAsync(); - - var idsToRemove = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); - - foreach (var ruleId in idsToRemove) - { - await RemoveSnapshotAsync(ruleId); - } - - await ruleEventRepository.RemoveAsync(appId); - } - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs index 6415863b8..2b3536f12 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs @@ -42,20 +42,6 @@ namespace Squidex.Domain.Apps.Entities.Schemas this.grainFactory = grainFactory; } - public override async Task RemoveAsync(Guid appId) - { - var index = grainFactory.GetGrain(appId); - - var idsToRemove = await index.GetSchemaIdsAsync(); - - foreach (var schemaId in idsToRemove) - { - await RemoveSnapshotAsync(schemaId); - } - - await index.ClearAsync(); - } - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) { switch (@event.Payload) diff --git a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs index 26e9dc123..4965bfd9e 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs @@ -23,7 +23,8 @@ namespace Squidex.Areas.Api.Controllers.Backups [ApiAuthorize] [ApiExceptionFilter] [ApiModelValidation(true)] - [SwaggerTag(nameof(Backups))] + [MustBeAdministrator] + [SwaggerIgnore] public class RestoreController : ApiController { private readonly IGrainFactory grainFactory; @@ -34,16 +35,8 @@ namespace Squidex.Areas.Api.Controllers.Backups this.grainFactory = grainFactory; } - /// - /// Get the restore jobs. - /// - /// - /// 200 => Restore job returned. - /// 404 => App not found. - /// [HttpGet] [Route("apps/restore/")] - [ProducesResponseType(typeof(RestoreJobDto), 200)] [ApiCosts(0)] public async Task GetJob() { @@ -63,13 +56,6 @@ namespace Squidex.Areas.Api.Controllers.Backups return Ok(response); } - /// - /// Start a new restore job. - /// - /// The request object. - /// - /// 204 => Backup started. - /// [HttpPost] [Route("apps/restore/")] [ApiCosts(0)] diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index 385fb4e4e..12e65d848 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -178,9 +178,6 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); - - services.AddSingletonAs() - .As(); } private static void AddBackupHandlers(this IServiceCollection services) @@ -194,9 +191,6 @@ namespace Squidex.Config.Domain services.AddTransientAs() .As(); - services.AddTransientAs() - .As(); - services.AddTransientAs() .As(); diff --git a/src/Squidex/Config/Orleans/SiloWrapper.cs b/src/Squidex/Config/Orleans/SiloWrapper.cs index 38292f58c..62139bb06 100644 --- a/src/Squidex/Config/Orleans/SiloWrapper.cs +++ b/src/Squidex/Config/Orleans/SiloWrapper.cs @@ -15,7 +15,6 @@ using Orleans; using Orleans.Configuration; using Orleans.Hosting; using Squidex.Config.Domain; -using Squidex.Domain.Apps.Entities.Backup; using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Rules; using Squidex.Infrastructure; @@ -63,7 +62,6 @@ namespace Squidex.Config.Orleans .EnableDirectClient() .AddIncomingGrainCallFilter() .AddStartupTask>() - .AddStartupTask>() .AddStartupTask>() .AddStartupTask>() .AddStartupTask((services, ct) => diff --git a/src/Squidex/app/features/administration/administration-area.component.html b/src/Squidex/app/features/administration/administration-area.component.html index e75dab4a4..6319cf409 100644 --- a/src/Squidex/app/features/administration/administration-area.component.html +++ b/src/Squidex/app/features/administration/administration-area.component.html @@ -12,6 +12,11 @@
Users
+
  • + +
    Restore
    +     +
    • diff --git a/src/Squidex/app/features/administration/declarations.ts b/src/Squidex/app/features/administration/declarations.ts index 451d9dc50..09279fe2c 100644 --- a/src/Squidex/app/features/administration/declarations.ts +++ b/src/Squidex/app/features/administration/declarations.ts @@ -11,6 +11,7 @@ export * from './guards/user-must-exist.guard'; export * from './guards/unset-user.guard'; export * from './pages/event-consumers/event-consumers-page.component'; +export * from './pages/restore/restore-page.component'; export * from './pages/users/user-page.component'; export * from './pages/users/users-page.component'; diff --git a/src/Squidex/app/features/administration/module.ts b/src/Squidex/app/features/administration/module.ts index c2d23b6b3..a97719a34 100644 --- a/src/Squidex/app/features/administration/module.ts +++ b/src/Squidex/app/features/administration/module.ts @@ -18,6 +18,7 @@ import { EventConsumersPageComponent, EventConsumersService, EventConsumersState, + RestorePageComponent, UnsetUserGuard, UserMustExistGuard, UserPageComponent, @@ -38,6 +39,10 @@ const routes: Routes = [ path: 'event-consumers', component: EventConsumersPageComponent }, + { + path: 'restore', + component: RestorePageComponent + }, { path: 'users', component: UsersPageComponent, @@ -69,6 +74,7 @@ const routes: Routes = [ declarations: [ AdministrationAreaComponent, EventConsumersPageComponent, + RestorePageComponent, UserPageComponent, UsersPageComponent ], diff --git a/src/Squidex/app/features/administration/pages/restore/restore-page.component.html b/src/Squidex/app/features/administration/pages/restore/restore-page.component.html new file mode 100644 index 000000000..82dbb493b --- /dev/null +++ b/src/Squidex/app/features/administration/pages/restore/restore-page.component.html @@ -0,0 +1,65 @@ + + + + + Restore Backup + + + + +
    +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +

    Last Restore Operation

    +
    + +
    + {{job.url}} +
    +
    +
    +
    +
    + {{row}} +
    +
    +
    +
    +
    + Started: {{job.started | sqxISODate}} +
    +
    + Stopped: {{job.stopped | sqxISODate}} +
    +
    +
    +
    +     + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    +     +     \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.scss b/src/Squidex/app/features/administration/pages/restore/restore-page.component.scss similarity index 97% rename from src/Squidex/app/features/apps/pages/restore-page.component.scss rename to src/Squidex/app/features/administration/pages/restore/restore-page.component.scss index 38c020056..ccd571a75 100644 --- a/src/Squidex/app/features/apps/pages/restore-page.component.scss +++ b/src/Squidex/app/features/administration/pages/restore/restore-page.component.scss @@ -62,6 +62,7 @@ h3 { } &-url { + @include truncate; line-height: 30px; } } \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.ts b/src/Squidex/app/features/administration/pages/restore/restore-page.component.ts similarity index 100% rename from src/Squidex/app/features/apps/pages/restore-page.component.ts rename to src/Squidex/app/features/administration/pages/restore/restore-page.component.ts diff --git a/src/Squidex/app/features/apps/declarations.ts b/src/Squidex/app/features/apps/declarations.ts index 8947fb5e9..880671f15 100644 --- a/src/Squidex/app/features/apps/declarations.ts +++ b/src/Squidex/app/features/apps/declarations.ts @@ -6,5 +6,4 @@ */ export * from './pages/apps-page.component'; -export * from './pages/onboarding-dialog.component'; -export * from './pages/restore-page.component'; \ No newline at end of file +export * from './pages/onboarding-dialog.component'; \ No newline at end of file diff --git a/src/Squidex/app/features/apps/module.ts b/src/Squidex/app/features/apps/module.ts index 51c4c718f..69c393429 100644 --- a/src/Squidex/app/features/apps/module.ts +++ b/src/Squidex/app/features/apps/module.ts @@ -12,17 +12,13 @@ import { SqxFrameworkModule, SqxSharedModule } from '@app/shared'; import { AppsPageComponent, - OnboardingDialogComponent, - RestorePageComponent + OnboardingDialogComponent } from './declarations'; const routes: Routes = [ { path: '', component: AppsPageComponent - }, { - path: 'restore', - component: RestorePageComponent } ]; @@ -34,8 +30,7 @@ const routes: Routes = [ ], declarations: [ AppsPageComponent, - OnboardingDialogComponent, - RestorePageComponent + OnboardingDialogComponent ] }) export class SqxFeatureAppsModule { } \ No newline at end of file diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.html b/src/Squidex/app/features/apps/pages/apps-page.component.html index b3aaf4bbe..461321025 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.html +++ b/src/Squidex/app/features/apps/pages/apps-page.component.html @@ -76,10 +76,6 @@
    -
    - Restore app from backup -
    - diff --git a/src/Squidex/app/features/apps/pages/apps-page.component.scss b/src/Squidex/app/features/apps/pages/apps-page.component.scss index af4c66b32..b789d434d 100644 --- a/src/Squidex/app/features/apps/pages/apps-page.component.scss +++ b/src/Squidex/app/features/apps/pages/apps-page.component.scss @@ -17,10 +17,6 @@ } } -.restore-link { - font-size: 0.8rem; -} - .card { & { margin-right: 1rem; diff --git a/src/Squidex/app/features/apps/pages/restore-page.component.html b/src/Squidex/app/features/apps/pages/restore-page.component.html deleted file mode 100644 index d4f481042..000000000 --- a/src/Squidex/app/features/apps/pages/restore-page.component.html +++ /dev/null @@ -1,59 +0,0 @@ - - -
    - -
    -
    -
    -
    -
    - -
    -
    - -
    -
    - -
    -
    - -
    -

    Last Restore Operation

    -
    - -
    - {{job.url}} -
    -
    -
    -
    -
    - {{row}} -
    -
    -
    -
    -
    - Started: {{job.started | sqxISODate}} -
    -
    - Stopped: {{job.stopped | sqxISODate}} -
    -
    -
    -
    -     - -
    -
    -
    -
    - -
    -
    - -
    -
    -
    -
    -
    \ No newline at end of file diff --git a/src/Squidex/app/theme/icomoon/demo.html b/src/Squidex/app/theme/icomoon/demo.html index c385962d3..3b417bb03 100644 --- a/src/Squidex/app/theme/icomoon/demo.html +++ b/src/Squidex/app/theme/icomoon/demo.html @@ -9,10 +9,26 @@
    -

    Font Name: icomoon (Glyphs: 94)

    +

    Font Name: icomoon (Glyphs: 95)

    Grid Size: 24

    +
    +
    + + + + icon-backup +
    +
    + + +
    +
    + liga: + +
    +
    diff --git a/src/Squidex/app/theme/icomoon/fonts/icomoon.eot b/src/Squidex/app/theme/icomoon/fonts/icomoon.eot index 7d47c86cfbdde7923012b3b1973b619ae6ae7d6b..7d637f8990f194cefdf7fb70f9311affd3c82407 100644 GIT binary patch delta 468 zcmZ2-gz?2uMz$A83=A$w6WPpIZWko&pXg97xQl^-VGj@|BMc&ODgh~inIXv8bCcg zIr+(n{wGp<7#QNW0Oe2QCRP+MR53;Yt*8O=74i~uQ)hZ{{s!{r0M)k@6 zl2kzQ49v`o$0w&Sb~8p#{=lfrI!z#)f97Thrhtfg25sgBW^D!&1_z)b#pmJVjiEL&p8%@*iPBu&wnHZ8RxRZf_VGj@|BgmYIPfqmzWPXZ)A?^cE{#b5eMFB%4V-y2J{0<;rAulmEb*2~RZy^5$P<>lLesKu{ zGf;*hF##mcz|72eY;p=?H)GV~4~)vJQw75KXKa>W3W(UO5gouRb2r^|V?4jjR|akt zkS2!fNj=wL^yDdV=8Q?3&&0JbP7X;pD!xy*N_QVnk`aiL7$X?OKnfX{CSObv-t3&X Glo0?S`B7K^ diff --git a/src/Squidex/app/theme/icomoon/fonts/icomoon.svg b/src/Squidex/app/theme/icomoon/fonts/icomoon.svg index 5f5cea4e1..ff07500d5 100644 --- a/src/Squidex/app/theme/icomoon/fonts/icomoon.svg +++ b/src/Squidex/app/theme/icomoon/fonts/icomoon.svg @@ -98,6 +98,7 @@ + diff --git a/src/Squidex/app/theme/icomoon/fonts/icomoon.ttf b/src/Squidex/app/theme/icomoon/fonts/icomoon.ttf index a1f6fc322e9d7c727343a6f5098a8c5d0297113c..c486ad6d3b49d3af49fe608f9197c9cd64652a75 100644 GIT binary patch delta 462 zcmZoT%y{GoV?6^S0|Ns$LjwadgMojrz7f9=?=GOo9w1Ig&P^=1ZFcJ=0|TQBke`#D zSX=<41%UhxAkC4UQ&rVv2CKKqCV~>=~fESq4ymlZiEjfgz3s z$XCh8Evd*`D$)YvYXJ51-KL@CxtsuX+1n5v8NKygGGcYqBpLn2~F?zBOqcZC>fpGqrn@bo2A`}_4nH!k3 z8B7=)fZB}2C<1 zF){eNxn;@$L8hB)fK!EmjcTNUow-?{I1mJyncG3w4Qp04fR!vWR#q|rVP}G}S!RDy z!R>f{o39Mqps-?KxZaiqrzac4nlmPE&WLScoctp0ByWYTobD|>IdI%1Z{8Pwh!Fq= CBygJm delta 302 zcmX?dgt6f;V?6^S0|Ns$LjwadgMojrz7f9=?@pk|9w1Ig&P^=1ZF=h&0|TQBke`#D zSX=<41%UhxAkC4UQ<)YzJH8CapTNKnU6YZTm?Eqkr_aC;%L9}*%K!>+GO;ED`7S`d zN=9x;Mb=W079c+ZWJXSYvSXtEC-YMb3~?WT3XbI_RunK)GDa~l#P0y|74i~uQ)hZ{ z{s!`I05!A~+D{890Jx>dUSfRdolNMej&5CbV> PV47^0B)oZk!cs;6x~Wf& diff --git a/src/Squidex/app/theme/icomoon/fonts/icomoon.woff b/src/Squidex/app/theme/icomoon/fonts/icomoon.woff index 9d034ac442e95e82885cd7334e713e229a76aaff..9e6cd0914ac54466be2d217987e21a86c6e0aab1 100644 GIT binary patch delta 520 zcmX?em~p~UMv-!VH#Y`G1|Ue9z`zZrT_$f#l$fX^T)!(hH?e?$fiVLpoCCtQ&2GI+ zPb>zBT>#s-8R)ZW{hk(!vozz`<^RAUCh!r20i89+gxSOAc( z0>Ye3tSK3}B^5xi5}^Jb5YAgF(vp*(3{>Zr093W0dU5_P$S(#OmIPD_RmTiuF*6^Z%){8t7(ICgqcZC>fpGqrn-4Gs zMAS2AGdD16GngoKaUDJy~H zM2*?ijZMH3!XhA*rY2^_Mq*41z|hQ;1ABW1sZ0Zk-C zH8~@C z3KRnx#@GhJGrc%}7vvY00DZCsu8tYVVrD)znTN5PF>3M*MrGEi0%80!HXmRNh}e7~ zDu7w$Zo28lcz&C&4BQ~o85pi7^<0P1lWpS68Iv}b#I-O^{u6(ccb{&R?mnO-D7=z3 JpG!Ez2mr_*Q(gc7 diff --git a/src/Squidex/app/theme/icomoon/selection.json b/src/Squidex/app/theme/icomoon/selection.json index 18bc8f403..67eb4be2f 100644 --- a/src/Squidex/app/theme/icomoon/selection.json +++ b/src/Squidex/app/theme/icomoon/selection.json @@ -1,6 +1,35 @@ { "IcoMoonType": "selection", "icons": [ + { + "icon": { + "paths": [ + "M512 128c212 0 384 172 384 384s-172 384-384 384c-88 0-170-30-234-80l60-60c50 34 110 54 174 54 166 0 298-132 298-298s-132-298-298-298-298 132-298 298h128l-172 170-170-170h128c0-212 172-384 384-384zM598 512c0 46-40 86-86 86s-86-40-86-86 40-86 86-86 86 40 86 86z" + ], + "attrs": [ + {} + ], + "isMulticolor": false, + "isMulticolor2": false, + "tags": [ + "settings_backup_restore" + ], + "grid": 24 + }, + "attrs": [ + {} + ], + "properties": { + "order": 1, + "id": 2, + "prevSize": 24, + "code": 59739, + "name": "backup" + }, + "setIdx": 1, + "setId": 3, + "iconIdx": 0 + }, { "icon": { "paths": [ @@ -26,9 +55,9 @@ "code": 59738, "name": "support" }, - "setIdx": 0, + "setIdx": 1, "setId": 3, - "iconIdx": 0 + "iconIdx": 2 }, { "icon": { @@ -55,7 +84,7 @@ "code": 59705, "name": "control-RichText" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 90 }, @@ -84,7 +113,7 @@ "code": 59710, "name": "download" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 91 }, @@ -113,7 +142,7 @@ "code": 59737, "name": "action-Medium" }, - "setIdx": 1, + "setIdx": 2, "setId": 2, "iconIdx": 0 }, @@ -142,7 +171,7 @@ "prevSize": 32, "code": 59729 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 44 }, @@ -172,7 +201,7 @@ "prevSize": 32, "code": 59726 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 45 }, @@ -204,7 +233,7 @@ "prevSize": 32, "code": 59727 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 46 }, @@ -233,7 +262,7 @@ "prevSize": 32, "code": 59724 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 47 }, @@ -271,7 +300,7 @@ "prevSize": 32, "code": 59722 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 48 }, @@ -300,7 +329,7 @@ "prevSize": 32, "code": 59652 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 49 }, @@ -329,7 +358,7 @@ "prevSize": 32, "code": 59653 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 50 }, @@ -358,7 +387,7 @@ "prevSize": 32, "code": 59654 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 51 }, @@ -387,7 +416,7 @@ "prevSize": 32, "code": 59655 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 52 }, @@ -416,7 +445,7 @@ "prevSize": 32, "code": 59656 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 53 }, @@ -445,7 +474,7 @@ "prevSize": 32, "code": 59657 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 54 }, @@ -474,7 +503,7 @@ "prevSize": 32, "code": 59658 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 55 }, @@ -503,7 +532,7 @@ "prevSize": 32, "code": 59659 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 56 }, @@ -532,7 +561,7 @@ "prevSize": 32, "code": 59660 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 57 }, @@ -561,7 +590,7 @@ "prevSize": 32, "code": 59661 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 58 }, @@ -590,7 +619,7 @@ "prevSize": 32, "code": 59662 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 59 }, @@ -619,7 +648,7 @@ "prevSize": 32, "code": 59663 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 60 }, @@ -648,7 +677,7 @@ "prevSize": 32, "code": 59664 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 61 }, @@ -683,7 +712,7 @@ "prevSize": 32, "code": 59665 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 62 }, @@ -712,7 +741,7 @@ "prevSize": 32, "code": 59666 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 63 }, @@ -741,7 +770,7 @@ "prevSize": 32, "code": 59667 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 64 }, @@ -770,7 +799,7 @@ "prevSize": 32, "code": 59668 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 65 }, @@ -799,7 +828,7 @@ "prevSize": 32, "code": 59669 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 66 }, @@ -828,7 +857,7 @@ "prevSize": 32, "code": 59670 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 67 }, @@ -857,7 +886,7 @@ "prevSize": 32, "code": 59671 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 68 }, @@ -886,7 +915,7 @@ "prevSize": 32, "code": 59672 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 69 }, @@ -916,7 +945,7 @@ "prevSize": 32, "code": 59713 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 70 }, @@ -945,7 +974,7 @@ "prevSize": 32, "code": 59673 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 71 }, @@ -974,7 +1003,7 @@ "prevSize": 32, "code": 59675 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 72 }, @@ -1003,7 +1032,7 @@ "prevSize": 32, "code": 59676 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 73 }, @@ -1032,7 +1061,7 @@ "prevSize": 32, "code": 59677 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 74 }, @@ -1061,7 +1090,7 @@ "prevSize": 32, "code": 59678 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 75 }, @@ -1090,7 +1119,7 @@ "prevSize": 32, "code": 59679 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 76 }, @@ -1119,7 +1148,7 @@ "prevSize": 32, "code": 59680 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 77 }, @@ -1157,7 +1186,7 @@ "prevSize": 32, "code": 59681 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 78 }, @@ -1186,7 +1215,7 @@ "prevSize": 32, "code": 59682 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 79 }, @@ -1215,7 +1244,7 @@ "prevSize": 32, "code": 59683 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 80 }, @@ -1244,7 +1273,7 @@ "prevSize": 32, "code": 59684 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 81 }, @@ -1273,7 +1302,7 @@ "prevSize": 32, "code": 59685 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 82 }, @@ -1302,7 +1331,7 @@ "prevSize": 32, "code": 59674 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 83 }, @@ -1331,7 +1360,7 @@ "prevSize": 32, "code": 59686 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 84 }, @@ -1360,7 +1389,7 @@ "prevSize": 32, "code": 59687 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 85 }, @@ -1389,7 +1418,7 @@ "prevSize": 32, "code": 59688 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 86 }, @@ -1428,7 +1457,7 @@ "prevSize": 28, "code": 59736 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 0 }, @@ -1470,7 +1499,7 @@ "prevSize": 28, "code": 59735 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 1 }, @@ -1508,7 +1537,7 @@ "prevSize": 28, "code": 59734 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 3 }, @@ -1538,7 +1567,7 @@ "code": 59733, "name": "exclamation" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 4 }, @@ -1568,7 +1597,7 @@ "prevSize": 28, "code": 59730 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 5 }, @@ -1598,7 +1627,7 @@ "code": 59725, "name": "action-Slack" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 6 }, @@ -1627,7 +1656,7 @@ "prevSize": 28, "code": 59723 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 7 }, @@ -1659,7 +1688,7 @@ "prevSize": 28, "code": 59721 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 8 }, @@ -1691,7 +1720,7 @@ "prevSize": 28, "code": 59711 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 9 }, @@ -1721,7 +1750,7 @@ "code": 59648, "name": "angle-down" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 10 }, @@ -1751,7 +1780,7 @@ "code": 59649, "name": "angle-left" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 11 }, @@ -1781,7 +1810,7 @@ "code": 59697, "name": "angle-right" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 12 }, @@ -1811,7 +1840,7 @@ "code": 59651, "name": "angle-up" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 13 }, @@ -1852,7 +1881,7 @@ "prevSize": 28, "code": 59717 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 14 }, @@ -1887,7 +1916,7 @@ "prevSize": 28, "code": 59720 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 15 }, @@ -1917,7 +1946,7 @@ "code": 59709, "name": "bug" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 16 }, @@ -1947,7 +1976,7 @@ "code": 59692, "name": "caret-down" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 17 }, @@ -1977,7 +2006,7 @@ "code": 59690, "name": "caret-left" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 18 }, @@ -2007,7 +2036,7 @@ "code": 59689, "name": "caret-right" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 19 }, @@ -2037,7 +2066,7 @@ "code": 59691, "name": "caret-up" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 20 }, @@ -2075,7 +2104,7 @@ "prevSize": 28, "code": 59718 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 21 }, @@ -2104,7 +2133,7 @@ "prevSize": 28, "code": 59702 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 22 }, @@ -2133,7 +2162,7 @@ "prevSize": 28, "code": 59703 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 23 }, @@ -2162,7 +2191,7 @@ "prevSize": 28, "code": 59704 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 24 }, @@ -2189,7 +2218,7 @@ "prevSize": 28, "code": 61450 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 25 }, @@ -2218,7 +2247,7 @@ "prevSize": 28, "code": 61641 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 26 }, @@ -2247,7 +2276,7 @@ "prevSize": 28, "code": 59698 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 27 }, @@ -2297,7 +2326,7 @@ "prevSize": 28, "code": 59719 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 28 }, @@ -2329,7 +2358,7 @@ "code": 59732, "name": "hour-glass" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 29 }, @@ -2362,7 +2391,7 @@ "code": 59731, "name": "spinner" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 30 }, @@ -2393,7 +2422,7 @@ "code": 59728, "name": "clock" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 31 }, @@ -2430,7 +2459,7 @@ "prevSize": 32, "code": 59650 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 32 }, @@ -2463,7 +2492,7 @@ "prevSize": 32, "code": 59850 }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 33 }, @@ -2496,7 +2525,7 @@ "code": 59715, "name": "elapsed" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 34 }, @@ -2526,7 +2555,7 @@ "code": 59707, "name": "google" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 35 }, @@ -2558,7 +2587,7 @@ "code": 59700, "name": "lock" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 36 }, @@ -2589,7 +2618,7 @@ "code": 59712, "name": "microsoft, action-AzureQueue" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 37 }, @@ -2619,7 +2648,7 @@ "code": 59695, "name": "pause" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 38 }, @@ -2649,7 +2678,7 @@ "code": 59696, "name": "play" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 39 }, @@ -2688,7 +2717,7 @@ "code": 59694, "name": "reset" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 40 }, @@ -2723,7 +2752,7 @@ "code": 59693, "name": "settings2" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 41 }, @@ -2757,7 +2786,7 @@ "code": 59716, "name": "timeout" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 42 }, @@ -2787,7 +2816,7 @@ "code": 59699, "name": "unlocked" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 43 }, @@ -2820,7 +2849,7 @@ "code": 59701, "name": "browser" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 87 }, @@ -2852,7 +2881,7 @@ "code": 59714, "name": "checkmark" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 88 }, @@ -2882,7 +2911,7 @@ "code": 59706, "name": "control-Stars" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 89 }, @@ -2911,7 +2940,7 @@ "code": 59708, "name": "info" }, - "setIdx": 2, + "setIdx": 3, "setId": 1, "iconIdx": 92 } diff --git a/src/Squidex/app/theme/icomoon/style.css b/src/Squidex/app/theme/icomoon/style.css index 78c745c05..8ef903e79 100644 --- a/src/Squidex/app/theme/icomoon/style.css +++ b/src/Squidex/app/theme/icomoon/style.css @@ -1,10 +1,10 @@ @font-face { font-family: 'icomoon'; - src: url('fonts/icomoon.eot?lx1t7v'); - src: url('fonts/icomoon.eot?lx1t7v#iefix') format('embedded-opentype'), - url('fonts/icomoon.ttf?lx1t7v') format('truetype'), - url('fonts/icomoon.woff?lx1t7v') format('woff'), - url('fonts/icomoon.svg?lx1t7v#icomoon') format('svg'); + src: url('fonts/icomoon.eot?za0y3d'); + src: url('fonts/icomoon.eot?za0y3d#iefix') format('embedded-opentype'), + url('fonts/icomoon.ttf?za0y3d') format('truetype'), + url('fonts/icomoon.woff?za0y3d') format('woff'), + url('fonts/icomoon.svg?za0y3d#icomoon') format('svg'); font-weight: normal; font-style: normal; } @@ -24,6 +24,9 @@ -moz-osx-font-smoothing: grayscale; } +.icon-backup:before { + content: "\e95b"; +} .icon-support:before { content: "\e95a"; } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs deleted file mode 100644 index 0a8c24350..000000000 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EnqueueAppToCleanerMiddlewareTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using FakeItEasy; -using Orleans; -using Squidex.Domain.Apps.Entities.Apps.Commands; -using Squidex.Infrastructure.Commands; -using Squidex.Infrastructure.Orleans; -using Xunit; - -namespace Squidex.Domain.Apps.Entities.Backup -{ - public class EnqueueAppToCleanerMiddlewareTests - { - private readonly IGrainFactory grainFactory = A.Fake(); - private readonly ICommandBus commandBus = A.Fake(); - private readonly IAppCleanerGrain index = A.Fake(); - private readonly Guid appId = Guid.NewGuid(); - private readonly EnqueueAppToCleanerMiddleware sut; - - public EnqueueAppToCleanerMiddlewareTests() - { - A.CallTo(() => grainFactory.GetGrain(SingleGrain.Id, null)) - .Returns(index); - - sut = new EnqueueAppToCleanerMiddleware(grainFactory); - } - - [Fact] - public async Task Should_enqueue_for_cleanup_on_archive() - { - var context = - new CommandContext(new ArchiveApp { AppId = appId }, commandBus) - .Complete(); - - await sut.HandleAsync(context); - - A.CallTo(() => index.EnqueueAppAsync(appId)) - .MustHaveHappened(); - } - } -} From 5a0bfdd287422432df892774266b699a76bf16a7 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 1 Aug 2018 19:25:57 +0200 Subject: [PATCH 18/23] User mapping. --- .../HandleRules/RuleEventFormatter.cs | 4 +- .../Scripting/ContentWrapper/JsonMapper.cs | 18 +-- .../Apps/BackupApps.cs | 132 ++++++++++++++++-- .../Assets/BackupAssets.cs | 58 +++----- .../Backup/BackupGrain.cs | 18 +-- .../Backup/BackupHandler.cs | 5 +- .../Backup/BackupReader.cs | 9 +- .../Backup/BackupSerializer.cs | 41 ++++++ .../Backup/BackupWriter.cs | 9 +- .../Backup/RestoreGrain.cs | 26 ++-- .../Schemas/BackupSchemas.cs | 2 +- .../CollectionExtensions.cs | 12 ++ src/Squidex.Infrastructure/RefTokenType.cs | 16 +++ src/Squidex.Infrastructure/States/IStore.cs | 4 - .../States/StoreExtensions.cs | 17 +++ .../EnrichWithActorCommandMiddleware.cs | 4 +- .../HandleRules/RuleEventFormatterTests.cs | 6 +- .../Contents/GraphQL/GraphQLTestBase.cs | 8 +- .../TestHelpers/HandlerTestBase.cs | 2 +- .../CollectionExtensionsTests.cs | 7 + .../States/PersistenceSnapshotTests.cs | 31 ++-- .../EnrichWithActorCommandMiddlewareTests.cs | 4 +- 22 files changed, 302 insertions(+), 131 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs create mode 100644 src/Squidex.Infrastructure/RefTokenType.cs diff --git a/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs b/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs index 778210249..e1c757c65 100644 --- a/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs +++ b/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs @@ -209,7 +209,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules { if (@event.Actor != null) { - if (@event.Actor.Type.Equals("client", StringComparison.OrdinalIgnoreCase)) + if (@event.Actor.Type.Equals(RefTokenType.Client, StringComparison.OrdinalIgnoreCase)) { return @event.Actor.ToString(); } @@ -227,7 +227,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules { if (@event.Actor != null) { - if (@event.Actor.Type.Equals("client", StringComparison.OrdinalIgnoreCase)) + if (@event.Actor.Type.Equals(RefTokenType.Client, StringComparison.OrdinalIgnoreCase)) { return @event.Actor.ToString(); } diff --git a/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs b/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs index ea3a19385..6ea3cd584 100644 --- a/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs +++ b/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs @@ -43,18 +43,18 @@ namespace Squidex.Domain.Apps.Core.Scripting.ContentWrapper case JTokenType.Object: return FromObject(value, engine); case JTokenType.Array: - { - var arr = (JArray)value; + { + var arr = (JArray)value; - var target = new JsValue[arr.Count]; + var target = new JsValue[arr.Count]; - for (var i = 0; i < arr.Count; i++) - { - target[i] = Map(arr[i], engine); - } + for (var i = 0; i < arr.Count; i++) + { + target[i] = Map(arr[i], engine); + } - return engine.Array.Construct(target); - } + return engine.Array.Construct(target); + } } throw new ArgumentException("Invalid json type.", nameof(value)); diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs index 398456c14..08138a045 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -11,32 +11,66 @@ using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Entities.Apps.Indexes; using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Events; using Squidex.Domain.Apps.Events.Apps; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; +using Squidex.Shared.Users; namespace Squidex.Domain.Apps.Entities.Apps { public sealed class BackupApps : BackupHandlerWithStore { + private const string UsersFile = "Users.json"; private readonly IGrainFactory grainFactory; - private readonly HashSet users = new HashSet(); + private readonly IUserResolver userResolver; + private readonly HashSet activeUsers = new HashSet(); + private readonly Dictionary usersWithEmail = new Dictionary(); + private readonly Dictionary userMapping = new Dictionary(); private bool isReserved; + private bool isActorAssigned; private AppCreated appCreated; public override string Name { get; } = "Apps"; - public BackupApps(IStore store, IGrainFactory grainFactory) + public BackupApps(IStore store, IGrainFactory grainFactory, IUserResolver userResolver) : base(store) { Guard.NotNull(grainFactory, nameof(grainFactory)); + Guard.NotNull(userResolver, nameof(userResolver)); this.grainFactory = grainFactory; + + this.userResolver = userResolver; + } + + public override async Task BackupEventAsync(Envelope @event, Guid appId, BackupWriter writer) + { + if (@event.Payload is AppContributorAssigned appContributorAssigned) + { + var userId = appContributorAssigned.ContributorId; + + if (!usersWithEmail.ContainsKey(userId)) + { + var user = await userResolver.FindByIdOrEmailAsync(userId); + + if (user != null) + { + usersWithEmail.Add(userId, user.Email); + } + } + } } - public async override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + public override Task BackupAsync(Guid appId, BackupWriter writer) + { + return WriterUsersAsync(writer); + } + + public async override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader, RefToken actor) { switch (@event.Payload) { @@ -44,31 +78,103 @@ namespace Squidex.Domain.Apps.Entities.Apps { this.appCreated = appCreated; - var index = grainFactory.GetGrain(SingleGrain.Id); + await ResolveUsersAsync(reader, actor); + await ReserveAppAsync(); + + break; + } - if (!(isReserved = await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name))) + case AppContributorAssigned contributorAssigned: + { + if (isActorAssigned) { - throw new BackupRestoreException("The app id or name is not available."); + contributorAssigned.ContributorId = MapUser(contributorAssigned.ContributorId, actor).Identifier; + } + else + { + isActorAssigned = true; + + contributorAssigned.ContributorId = actor.Identifier; } + activeUsers.Add(contributorAssigned.ContributorId); break; } - case AppContributorAssigned contributorAssigned: - users.Add(contributorAssigned.ContributorId); - break; - case AppContributorRemoved contributorRemoved: - users.Remove(contributorRemoved.ContributorId); - break; + { + contributorRemoved.ContributorId = MapUser(contributorRemoved.ContributorId, actor).Identifier; + + activeUsers.Remove(contributorRemoved.ContributorId); + break; + } + } + + if (@event.Payload is SquidexEvent squidexEvent) + { + squidexEvent.Actor = MapUser(squidexEvent.Actor.Identifier, actor); + } + } + + private async Task ReserveAppAsync() + { + var index = grainFactory.GetGrain(SingleGrain.Id); + + if (!(isReserved = await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name))) + { + throw new BackupRestoreException("The app id or name is not available."); + } + } + + private RefToken MapUser(string userId, RefToken fallback) + { + return userMapping.GetOrAdd(userId, fallback); + } + + private async Task ResolveUsersAsync(BackupReader reader, RefToken actor) + { + await ReadUsersAsync(reader); + + foreach (var kvp in usersWithEmail) + { + var user = await userResolver.FindByIdOrEmailAsync(kvp.Value); + + if (user != null) + { + userMapping[kvp.Key] = new RefToken(RefTokenType.Subject, user.Id); + } + else + { + userMapping[kvp.Key] = actor; + } } } + private async Task ReadUsersAsync(BackupReader reader) + { + await reader.ReadAttachmentAsync(UsersFile, stream => + { + stream.SerializeAsJson(usersWithEmail); + + return TaskHelper.Done; + }); + } + + private Task WriterUsersAsync(BackupWriter writer) + { + return writer.WriteAttachmentAsync(UsersFile, stream => + { + stream.SerializeAsJson(usersWithEmail); + + return TaskHelper.Done; + }); + } + public override async Task CompleteRestoreAsync(Guid appId, BackupReader reader) { await grainFactory.GetGrain(SingleGrain.Id).AddAppAsync(appCreated.AppId.Id, appCreated.AppId.Name); - foreach (var user in users) + foreach (var user in activeUsers) { await grainFactory.GetGrain(user).AddAppAsync(appCreated.AppId.Id); } diff --git a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index b6d59f386..39573c6e6 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -7,9 +7,7 @@ using System; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; -using Newtonsoft.Json; using Squidex.Domain.Apps.Entities.Assets.Repositories; using Squidex.Domain.Apps.Entities.Assets.State; using Squidex.Domain.Apps.Entities.Backup; @@ -25,64 +23,52 @@ namespace Squidex.Domain.Apps.Entities.Assets { public sealed class BackupAssets : BackupHandlerWithStore { - private static readonly JsonSerializer Serializer = JsonSerializer.Create(); + private const string TagsFile = "AssetTags.json"; private readonly HashSet assetIds = new HashSet(); private readonly IAssetStore assetStore; private readonly IAssetRepository assetRepository; private readonly ITagService tagService; - private readonly IEventDataFormatter eventDataFormatter; public override string Name { get; } = "Assets"; public BackupAssets(IStore store, - IEventDataFormatter eventDataFormatter, IAssetStore assetStore, IAssetRepository assetRepository, ITagService tagService) : base(store) { - Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter)); Guard.NotNull(assetStore, nameof(assetStore)); Guard.NotNull(assetRepository, nameof(assetRepository)); Guard.NotNull(tagService, nameof(tagService)); - this.eventDataFormatter = eventDataFormatter; this.assetStore = assetStore; this.assetRepository = assetRepository; this.tagService = tagService; } - public override Task BackupEventAsync(EventData @event, Guid appId, BackupWriter writer) + public override Task BackupAsync(Guid appId, BackupWriter writer) { - if (@event.Type == "AssetCreatedEvent" || - @event.Type == "AssetUpdatedEvent") - { - var parsedEvent = eventDataFormatter.Parse(@event); + return BackupTagsAsync(appId, writer); + } - switch (parsedEvent.Payload) - { - case AssetCreated assetCreated: - return WriteAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, writer); - case AssetUpdated assetUpdated: - return WriteAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, writer); - } + public override Task BackupEventAsync(Envelope @event, Guid appId, BackupWriter writer) + { + switch (@event.Payload) + { + case AssetCreated assetCreated: + return WriteAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, writer); + case AssetUpdated assetUpdated: + return WriteAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, writer); } return TaskHelper.Done; } - public override Task BackupAsync(Guid appId, BackupWriter writer) - { - return BackupTagsAsync(appId, writer); - } - - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader, RefToken actor) { switch (@event.Payload) { case AssetCreated assetCreated: - assetIds.Add(assetCreated.AssetId); - return ReadAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, reader); case AssetUpdated assetUpdated: return ReadAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, reader); @@ -100,27 +86,21 @@ namespace Squidex.Domain.Apps.Entities.Assets private Task RestoreTagsAsync(Guid appId, BackupReader reader) { - return reader.ReadAttachmentAsync("AssetTags.json", async stream => + return reader.ReadAttachmentAsync(TagsFile, stream => { - using (var textReader = new StreamReader(stream)) - { - var tags = (TagSet)Serializer.Deserialize(textReader, typeof(TagSet)); + var tags = stream.DeserializeAsJson(); - await tagService.RebuildTagsAsync(appId, TagGroups.Assets, tags); - } + return tagService.RebuildTagsAsync(appId, TagGroups.Assets, tags); }); } private Task BackupTagsAsync(Guid appId, BackupWriter writer) { - return writer.WriteAttachmentAsync("AssetTags.json", async stream => + return writer.WriteAttachmentAsync(TagsFile, async stream => { var tags = await tagService.GetExportableTagsAsync(appId, TagGroups.Assets); - using (var textWriter = new StreamWriter(stream)) - { - Serializer.Serialize(textWriter, tags); - } + stream.SerializeAsJson(tags); }); } @@ -134,6 +114,8 @@ namespace Squidex.Domain.Apps.Entities.Assets private Task ReadAssetAsync(Guid assetId, long fileVersion, BackupReader reader) { + assetIds.Add(assetId); + return reader.ReadAttachmentAsync(GetName(assetId, fileVersion), async stream => { try diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index 491c2b05f..b1f87e841 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -30,18 +30,18 @@ namespace Squidex.Domain.Apps.Entities.Backup { private const int MaxBackups = 10; private static readonly Duration UpdateDuration = Duration.FromSeconds(1); - private readonly IClock clock; private readonly IAssetStore assetStore; - private readonly IEventDataFormatter eventDataFormatter; + private readonly IBackupArchiveLocation backupArchiveLocation; + private readonly IClock clock; private readonly IEnumerable handlers; - private readonly ISemanticLog log; + private readonly IEventDataFormatter eventDataFormatter; private readonly IEventStore eventStore; - private readonly IBackupArchiveLocation backupArchiveLocation; + private readonly ISemanticLog log; private readonly IStore store; private CancellationTokenSource currentTask; private BackupStateJob currentJob; - private Guid appId; private BackupState state = new BackupState(); + private Guid appId; private IPersistence persistence; public BackupGrain( @@ -141,15 +141,15 @@ namespace Squidex.Domain.Apps.Entities.Backup { using (var writer = new BackupWriter(stream)) { - await eventStore.QueryAsync(async @event => + await eventStore.QueryAsync(async storedEvent => { - var eventData = @event.Data; + var @event = eventDataFormatter.Parse(storedEvent.Data); - writer.WriteEvent(@event); + writer.WriteEvent(storedEvent); foreach (var handler in handlers) { - await handler.BackupEventAsync(eventData, appId, writer); + await handler.BackupEventAsync(@event, appId, writer); } job.HandledEvents = writer.WrittenEvents; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs index 645792727..d640535e0 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs @@ -7,6 +7,7 @@ using System; using System.Threading.Tasks; +using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Tasks; @@ -16,12 +17,12 @@ namespace Squidex.Domain.Apps.Entities.Backup { public abstract string Name { get; } - public virtual Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + public virtual Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader, RefToken actor) { return TaskHelper.Done; } - public virtual Task BackupEventAsync(EventData @event, Guid appId, BackupWriter writer) + public virtual Task BackupEventAsync(Envelope @event, Guid appId, BackupWriter writer) { return TaskHelper.Done; } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs index bb43a04ab..8970cf26f 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs @@ -9,7 +9,6 @@ using System; using System.IO; using System.IO.Compression; using System.Threading.Tasks; -using Newtonsoft.Json; using Squidex.Domain.Apps.Entities.Backup.Archive; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; @@ -18,7 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class BackupReader : DisposableObjectBase { - private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); private readonly ZipArchive archive; private int readEvents; private int readAttachments; @@ -81,12 +79,9 @@ namespace Squidex.Domain.Apps.Entities.Backup using (var stream = eventEntry.Open()) { - using (var textReader = new StreamReader(stream)) - { - var storedEvent = (StoredEvent)JsonSerializer.Deserialize(textReader, typeof(StoredEvent)); + var storedEvent = stream.DeserializeAsJson(); - await handler(storedEvent); - } + await handler(storedEvent); } readEvents++; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs new file mode 100644 index 000000000..b638f9da4 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs @@ -0,0 +1,41 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.IO; +using Newtonsoft.Json; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public static class BackupSerializer + { + private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); + + public static void SerializeAsJson(this Stream stream, T value) + { + using (var writer = new StreamWriter(stream)) + { + JsonSerializer.Serialize(writer, value); + } + } + + public static T DeserializeAsJson(this Stream stream) + { + using (var reader = new StreamReader(stream)) + { + return (T)JsonSerializer.Deserialize(reader, typeof(T)); + } + } + + public static void DeserializeAsJson(this Stream stream, T result) + { + using (var reader = new StreamReader(stream)) + { + JsonSerializer.Populate(reader, result); + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs index 9ee875e3b..49c45f94a 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs @@ -9,7 +9,6 @@ using System; using System.IO; using System.IO.Compression; using System.Threading.Tasks; -using Newtonsoft.Json; using Squidex.Domain.Apps.Entities.Backup.Archive; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; @@ -18,7 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class BackupWriter : DisposableObjectBase { - private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); private readonly ZipArchive archive; private int writtenEvents; private int writtenAttachments; @@ -63,14 +61,13 @@ namespace Squidex.Domain.Apps.Entities.Backup public void WriteEvent(StoredEvent storedEvent) { + Guard.NotNull(storedEvent, nameof(storedEvent)); + var eventEntry = archive.CreateEntry(ArchiveHelper.GetEventPath(writtenEvents)); using (var stream = eventEntry.Open()) { - using (var textWriter = new StreamWriter(stream)) - { - JsonSerializer.Serialize(textWriter, storedEvent); - } + stream.SerializeAsJson(storedEvent); } writtenEvents++; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index 61acfa026..6200c7bf5 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -26,15 +26,15 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class RestoreGrain : GrainOfString, IRestoreGrain { - private readonly IClock clock; private readonly IAssetStore assetStore; + private readonly IBackupArchiveLocation backupArchiveLocation; + private readonly IClock clock; + private readonly IEnumerable handlers; + private readonly IEventStore eventStore; private readonly IEventDataFormatter eventDataFormatter; private readonly IGrainFactory grainFactory; private readonly ISemanticLog log; - private readonly IEventStore eventStore; - private readonly IBackupArchiveLocation backupArchiveLocation; private readonly IStore store; - private readonly IEnumerable handlers; private RefToken actor; private RestoreState state = new RestoreState(); private IPersistence persistence; @@ -78,7 +78,7 @@ namespace Squidex.Domain.Apps.Entities.Backup public override async Task OnActivateAsync(string key) { - actor = new RefToken("subject", key); + actor = new RefToken(RefTokenType.Subject, key); persistence = store.WithSnapshots(GetType(), key, s => state = s); @@ -255,8 +255,6 @@ namespace Squidex.Domain.Apps.Entities.Backup private async Task ReadEventsAsync(BackupReader reader) { - var isOwnerAdded = false; - await reader.ReadEventsAsync(async (storedEvent) => { var @event = eventDataFormatter.Parse(storedEvent.Data); @@ -270,24 +268,16 @@ namespace Squidex.Domain.Apps.Entities.Backup { CurrentJob.AppId = appCreated.AppId.Id; } - else if (@event.Payload is AppContributorAssigned appContributorAssigned) - { - if (!isOwnerAdded) - { - isOwnerAdded = true; - - appContributorAssigned.ContributorId = actor.Identifier; - } - } foreach (var handler in handlers) { - await handler.RestoreEventAsync(@event, CurrentJob.AppId, reader); + await handler.RestoreEventAsync(@event, CurrentJob.AppId, reader, actor); } var eventData = eventDataFormatter.ToEventData(@event, @event.Headers.CommitId()); + var eventCommit = new List { eventData }; - await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, new List { eventData }); + await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, eventCommit); Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments.", true); }); diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs index 2b3536f12..600d8e0e1 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs @@ -42,7 +42,7 @@ namespace Squidex.Domain.Apps.Entities.Schemas this.grainFactory = grainFactory; } - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader, RefToken actor) { switch (@event.Payload) { diff --git a/src/Squidex.Infrastructure/CollectionExtensions.cs b/src/Squidex.Infrastructure/CollectionExtensions.cs index 8e021e970..61513594d 100644 --- a/src/Squidex.Infrastructure/CollectionExtensions.cs +++ b/src/Squidex.Infrastructure/CollectionExtensions.cs @@ -167,6 +167,18 @@ namespace Squidex.Infrastructure return result; } + public static TValue GetOrAdd(this IDictionary dictionary, TKey key, TValue fallback) + { + if (!dictionary.TryGetValue(key, out var result)) + { + result = fallback; + + dictionary.Add(key, result); + } + + return result; + } + public static TValue GetOrAdd(this IDictionary dictionary, TKey key, Func creator) { if (!dictionary.TryGetValue(key, out var result)) diff --git a/src/Squidex.Infrastructure/RefTokenType.cs b/src/Squidex.Infrastructure/RefTokenType.cs new file mode 100644 index 000000000..c8ee8944e --- /dev/null +++ b/src/Squidex.Infrastructure/RefTokenType.cs @@ -0,0 +1,16 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Infrastructure +{ + public static class RefTokenType + { + public const string Subject = "subject"; + + public const string Client = "client"; + } +} diff --git a/src/Squidex.Infrastructure/States/IStore.cs b/src/Squidex.Infrastructure/States/IStore.cs index f8aa2cddd..57b33552e 100644 --- a/src/Squidex.Infrastructure/States/IStore.cs +++ b/src/Squidex.Infrastructure/States/IStore.cs @@ -20,9 +20,5 @@ namespace Squidex.Infrastructure.States IPersistence WithSnapshotsAndEventSourcing(Type owner, TKey key, Func applySnapshot, Func, Task> applyEvent); ISnapshotStore GetSnapshotStore(); - - Task ClearSnapshotsAsync(); - - Task RemoveSnapshotAsync(TKey key); } } diff --git a/src/Squidex.Infrastructure/States/StoreExtensions.cs b/src/Squidex.Infrastructure/States/StoreExtensions.cs index 1164807ad..17b9bf6f8 100644 --- a/src/Squidex.Infrastructure/States/StoreExtensions.cs +++ b/src/Squidex.Infrastructure/States/StoreExtensions.cs @@ -58,5 +58,22 @@ namespace Squidex.Infrastructure.States { return store.WithSnapshotsAndEventSourcing(typeof(TOwner), key, applySnapshot.ToAsync(), applyEvent.ToAsync()); } + + public static Task ClearSnapshotsAsync(this IStore store) + { + return store.GetSnapshotStore().ClearAsync(); + } + + public static Task RemoveSnapshotAsync(this IStore store, TKey key) + { + return store.GetSnapshotStore().RemoveAsync(key); + } + + public static async Task GetSnapshotAsync(this IStore store, TKey key) + { + var result = await store.GetSnapshotStore().ReadAsync(key); + + return result.Value; + } } } diff --git a/src/Squidex/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddleware.cs b/src/Squidex/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddleware.cs index f8e16d354..33690ade1 100644 --- a/src/Squidex/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddleware.cs +++ b/src/Squidex/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddleware.cs @@ -56,14 +56,14 @@ namespace Squidex.Pipeline.CommandMiddlewares { var subjectId = httpContextAccessor.HttpContext.User.OpenIdSubject(); - return subjectId == null ? null : new RefToken("subject", subjectId); + return subjectId == null ? null : new RefToken(RefTokenType.Subject, subjectId); } private RefToken FindActorFromClient() { var clientId = httpContextAccessor.HttpContext.User.OpenIdClientId(); - return clientId == null ? null : new RefToken("client", clientId); + return clientId == null ? null : new RefToken(RefTokenType.Client, clientId); } } } diff --git a/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs b/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs index 0bec1cd1d..b7ff530fd 100644 --- a/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs +++ b/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs @@ -106,7 +106,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Fact] public void Should_format_email_and_display_name_from_user() { - var @event = new EnrichedContentEvent { User = user, Actor = new RefToken("subject", "123") }; + var @event = new EnrichedContentEvent { User = user, Actor = new RefToken(RefTokenType.Subject, "123") }; var result = sut.Format("From $USER_NAME ($USER_EMAIL)", @event); @@ -116,7 +116,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Fact] public void Should_return_undefined_if_user_is_not_found() { - var @event = new EnrichedContentEvent { Actor = new RefToken("subject", "123") }; + var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Subject, "123") }; var result = sut.Format("From $USER_NAME ($USER_EMAIL)", @event); @@ -126,7 +126,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Fact] public void Should_format_email_and_display_name_from_client() { - var @event = new EnrichedContentEvent { Actor = new RefToken("client", "android") }; + var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "android") }; var result = sut.Format("From $USER_NAME ($USER_EMAIL)", @event); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLTestBase.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLTestBase.cs index 44190e333..f25465a81 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLTestBase.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLTestBase.cs @@ -141,9 +141,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.GraphQL Id = id, Version = 1, Created = now, - CreatedBy = new RefToken("subject", "user1"), + CreatedBy = new RefToken(RefTokenType.Subject, "user1"), LastModified = now, - LastModifiedBy = new RefToken("subject", "user2"), + LastModifiedBy = new RefToken(RefTokenType.Subject, "user2"), Data = data }; @@ -159,9 +159,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.GraphQL Id = id, Version = 1, Created = now, - CreatedBy = new RefToken("subject", "user1"), + CreatedBy = new RefToken(RefTokenType.Subject, "user1"), LastModified = now, - LastModifiedBy = new RefToken("subject", "user2"), + LastModifiedBy = new RefToken(RefTokenType.Subject, "user2"), FileName = "MyFile.png", FileSize = 1024, FileVersion = 123, diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs b/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs index 60c257bb8..d10ec2201 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs @@ -27,7 +27,7 @@ namespace Squidex.Domain.Apps.Entities.TestHelpers private readonly IPersistence persistence1 = A.Fake>(); private readonly IPersistence persistence2 = A.Fake(); - protected RefToken User { get; } = new RefToken("subject", Guid.NewGuid().ToString()); + protected RefToken User { get; } = new RefToken(RefTokenType.Subject, Guid.NewGuid().ToString()); protected Guid AppId { get; } = Guid.NewGuid(); diff --git a/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs b/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs index f1746a03f..d540ff429 100644 --- a/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs +++ b/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs @@ -71,6 +71,13 @@ namespace Squidex.Infrastructure [Fact] public void GetOrAdd_should_return_default_and_add_it_if_key_not_exists() + { + Assert.Equal(24, valueDictionary.GetOrAdd(12, 24)); + Assert.Equal(24, valueDictionary[12]); + } + + [Fact] + public void GetOrAdd_should_return_default_and_add_it_with_fallback_if_key_not_exists() { Assert.Equal(24, valueDictionary.GetOrAdd(12, x => 24)); Assert.Equal(24, valueDictionary[12]); diff --git a/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs b/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs index 392f8efff..06701c721 100644 --- a/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs +++ b/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs @@ -33,15 +33,6 @@ namespace Squidex.Infrastructure.States sut = new Store(eventStore, eventDataFormatter, services, streamNameResolver); } - [Fact] - public async Task Should_call_snapshot_store_on_clear() - { - await sut.ClearSnapshotsAsync(); - - A.CallTo(() => snapshotStore.ClearAsync()) - .MustHaveHappened(); - } - [Fact] public async Task Should_read_from_store() { @@ -161,10 +152,19 @@ namespace Squidex.Infrastructure.States .MustHaveHappened(); } + [Fact] + public async Task Should_call_snapshot_store_on_clear() + { + await sut.ClearSnapshotsAsync(); + + A.CallTo(() => snapshotStore.ClearAsync()) + .MustHaveHappened(); + } + [Fact] public async Task Should_delete_snapshot_but_not_events_when_deleted_from_store() { - await sut.RemoveSnapshotAsync(key); + await sut.RemoveSnapshotAsync(key); A.CallTo(() => eventStore.DeleteStreamAsync(A.Ignored)) .MustNotHaveHappened(); @@ -172,5 +172,16 @@ namespace Squidex.Infrastructure.States A.CallTo(() => snapshotStore.RemoveAsync(key)) .MustHaveHappened(); } + + [Fact] + public async Task Should_get_snapshot() + { + A.CallTo(() => snapshotStore.ReadAsync(key)) + .Returns((123, -1)); + + var result = await sut.GetSnapshotAsync(key); + + Assert.Equal(123, result); + } } } \ No newline at end of file diff --git a/tests/Squidex.Tests/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddlewareTests.cs b/tests/Squidex.Tests/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddlewareTests.cs index ce0ce1d9a..28977d34b 100644 --- a/tests/Squidex.Tests/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddlewareTests.cs +++ b/tests/Squidex.Tests/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddlewareTests.cs @@ -66,7 +66,7 @@ namespace Squidex.Pipeline.CommandMiddlewares await sut.HandleAsync(context); - Assert.Equal(new RefToken("subject", "me"), command.Actor); + Assert.Equal(new RefToken(RefTokenType.Subject, "me"), command.Actor); } [Fact] @@ -79,7 +79,7 @@ namespace Squidex.Pipeline.CommandMiddlewares await sut.HandleAsync(context); - Assert.Equal(new RefToken("client", "my-client"), command.Actor); + Assert.Equal(new RefToken(RefTokenType.Client, "my-client"), command.Actor); } [Fact] From cbc6f3600c62b9d74715745cd1a86269d6625b9b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 1 Aug 2018 20:16:43 +0200 Subject: [PATCH 19/23] GuidMapper --- .../Backup/BackupHandlerWithStore.cs | 2 +- .../Backup/GuidMapper.cs | 82 ++++++++++++++++++ .../Contents/BackupContents.cs | 2 +- .../Rules/BackupRules.cs | 2 +- ...eamTests.cs => BackupReaderWriterTests.cs} | 2 +- .../Backup/GuidMapperTests.cs | 86 +++++++++++++++++++ 6 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs rename tests/Squidex.Domain.Apps.Entities.Tests/Backup/{EventStreamTests.cs => BackupReaderWriterTests.cs} (98%) create mode 100644 tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs index 67fd56e0f..d6a2eba0d 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs @@ -28,7 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Backup protected Task RemoveSnapshotAsync(Guid id) { - return store.RemoveSnapshotAsync(id); + return store.RemoveSnapshotAsync(id); } protected async Task RebuildManyAsync(IEnumerable ids, Func action) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs b/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs new file mode 100644 index 000000000..7c97f4cd5 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs @@ -0,0 +1,82 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using Newtonsoft.Json.Linq; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public static class GuidMapper + { + private static readonly int GuidLength = Guid.Empty.ToString().Length; + + public static void GenerateNewGuid(JToken jToken, Dictionary guids) + { + if (jToken.Type == JTokenType.Object) + { + GenerateNewGuid((JObject)jToken, guids); + } + } + + private static void GenerateNewGuid(JObject jObject, Dictionary guids) + { + foreach (var kvp in jObject) + { + switch (kvp.Value.Type) + { + case JTokenType.String: + ReplaceGuidString(jObject, guids, kvp); + break; + case JTokenType.Guid: + ReplaceGuid(jObject, guids, kvp); + break; + case JTokenType.Object: + GenerateNewGuid((JObject)kvp.Value, guids); + break; + } + } + } + + private static void ReplaceGuidString(JObject jObject, Dictionary guids, KeyValuePair kvp) + { + var value = kvp.Value.ToString(); + + if (value.Length == GuidLength) + { + if (Guid.TryParse(value, out var guid)) + { + var newGuid = guids.GetOrAdd(guid, GuidGenerator); + + jObject.Property(kvp.Key).Value = newGuid.ToString(); + } + } + else if (value.Length > GuidLength && value[GuidLength] == ',') + { + if (Guid.TryParse(value.Substring(0, GuidLength), out var guid)) + { + var newGuid = guids.GetOrAdd(guid, GuidGenerator); + + jObject.Property(kvp.Key).Value = newGuid + value.Substring(GuidLength); + } + } + } + + private static void ReplaceGuid(JObject jObject, Dictionary guids, KeyValuePair kvp) + { + var newGuid = guids.GetOrAdd((Guid)kvp.Value, GuidGenerator); + + jObject.Property(kvp.Key).Value = newGuid; + } + + private static Guid GuidGenerator(Guid key) + { + return Guid.NewGuid(); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs index 40c2eb0d4..38b905248 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs @@ -34,7 +34,7 @@ namespace Squidex.Domain.Apps.Entities.Contents this.contentRepository = contentRepository; } - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader, RefToken actor) { switch (@event.Payload) { diff --git a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs index a6260d6c2..4700dc163 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs @@ -40,7 +40,7 @@ namespace Squidex.Domain.Apps.Entities.Rules this.ruleEventRepository = ruleEventRepository; } - public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader) + public override Task RestoreEventAsync(Envelope @event, Guid appId, BackupReader reader, RefToken actor) { switch (@event.Payload) { diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs similarity index 98% rename from tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs rename to tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs index 0085d1852..fb80ca92d 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/EventStreamTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs @@ -15,7 +15,7 @@ using Xunit; namespace Squidex.Domain.Apps.Entities.Backup { - public class EventStreamTests + public class BackupReaderWriterTests { [Fact] public async Task Should_write_and_read_events() diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs new file mode 100644 index 000000000..a482cde0a --- /dev/null +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs @@ -0,0 +1,86 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public class GuidMapperTests + { + [Fact] + public void Should_map_guid() + { + var m = new Dictionary(); + + var id1 = Guid.NewGuid(); + var id2 = Guid.NewGuid(); + + var obj = + new JObject( + new JProperty("k", + new JObject( + new JProperty("v1", id1), + new JProperty("v2", id1), + new JProperty("v3", id2)))); + + GuidMapper.GenerateNewGuid(obj, m); + + Assert.Equal(m[id1], obj["k"].Value("v1")); + Assert.Equal(m[id1], obj["k"].Value("v2")); + Assert.Equal(m[id2], obj["k"].Value("v3")); + } + + [Fact] + public void Should_map_guid_string() + { + var m = new Dictionary(); + + var id1 = Guid.NewGuid(); + var id2 = Guid.NewGuid(); + + var obj = + new JObject( + new JProperty("k", + new JObject( + new JProperty("v1", id1.ToString()), + new JProperty("v2", id1.ToString()), + new JProperty("v3", id2.ToString())))); + + GuidMapper.GenerateNewGuid(obj, m); + + Assert.Equal(m[id1].ToString(), obj["k"].Value("v1")); + Assert.Equal(m[id1].ToString(), obj["k"].Value("v2")); + Assert.Equal(m[id2].ToString(), obj["k"].Value("v3")); + } + + [Fact] + public void Should_map_named_id() + { + var m = new Dictionary(); + + var id1 = Guid.NewGuid(); + var id2 = Guid.NewGuid(); + + var obj = + new JObject( + new JProperty("k", + new JObject( + new JProperty("v1", $"{id1},v1"), + new JProperty("v2", $"{id1},v2"), + new JProperty("v3", $"{id2},v3")))); + + GuidMapper.GenerateNewGuid(obj, m); + + Assert.Equal($"{m[id1].ToString()},v1", obj["k"].Value("v1")); + Assert.Equal($"{m[id1].ToString()},v2", obj["k"].Value("v2")); + Assert.Equal($"{m[id2].ToString()},v3", obj["k"].Value("v3")); + } + } +} From 65c3cfbd88dc1dc269f0a4ad678c974c6864c2bc Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 19 Aug 2018 12:13:05 +0200 Subject: [PATCH 20/23] Temp --- .../Apps/BackupApps.cs | 20 +-- .../Assets/BackupAssets.cs | 23 ++- .../Backup/BackupReader.cs | 66 +++++++- .../Backup/BackupSerializer.cs | 41 ----- .../Backup/BackupWriter.cs | 37 ++++- .../Backup/GuidMapper.cs | 146 ++++++++++++++---- .../Backup/IRestoreGrain.cs | 2 +- .../Backup/RestoreGrain.cs | 10 +- .../Backup/State/RestoreStateJob.cs | 3 + .../ICleanableAppGrain.cs | 17 -- .../States/DefaultStreamNameResolver.cs | 23 +++ .../States/IStreamNameResolver.cs | 2 + .../Backups/Models/RestoreRequest.cs | 7 + .../Controllers/Backups/RestoreController.cs | 2 +- .../Backup/BackupReaderWriterTests.cs | 47 +++++- .../Backup/GuidMapperTests.cs | 131 ++++++++++++---- .../States/DefaultStreamNameResolverTests.cs | 30 ++++ 17 files changed, 445 insertions(+), 162 deletions(-) delete mode 100644 src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs delete mode 100644 src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs index 08138a045..c54ae17a5 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Newtonsoft.Json.Linq; using Orleans; using Squidex.Domain.Apps.Entities.Apps.Indexes; using Squidex.Domain.Apps.Entities.Backup; @@ -17,7 +18,6 @@ using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.States; -using Squidex.Infrastructure.Tasks; using Squidex.Shared.Users; namespace Squidex.Domain.Apps.Entities.Apps @@ -28,8 +28,8 @@ namespace Squidex.Domain.Apps.Entities.Apps private readonly IGrainFactory grainFactory; private readonly IUserResolver userResolver; private readonly HashSet activeUsers = new HashSet(); - private readonly Dictionary usersWithEmail = new Dictionary(); - private readonly Dictionary userMapping = new Dictionary(); + private Dictionary usersWithEmail = new Dictionary(); + private Dictionary userMapping = new Dictionary(); private bool isReserved; private bool isActorAssigned; private AppCreated appCreated; @@ -152,22 +152,16 @@ namespace Squidex.Domain.Apps.Entities.Apps private async Task ReadUsersAsync(BackupReader reader) { - await reader.ReadAttachmentAsync(UsersFile, stream => - { - stream.SerializeAsJson(usersWithEmail); + var json = await reader.ReadJsonAttachmentAsync(UsersFile); - return TaskHelper.Done; - }); + usersWithEmail = json.ToObject>(); } private Task WriterUsersAsync(BackupWriter writer) { - return writer.WriteAttachmentAsync(UsersFile, stream => - { - stream.SerializeAsJson(usersWithEmail); + var json = JObject.FromObject(usersWithEmail); - return TaskHelper.Done; - }); + return writer.WriteJsonAsync(UsersFile, json); } public override async Task CompleteRestoreAsync(Guid appId, BackupReader reader) diff --git a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index 39573c6e6..a63d894b6 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Newtonsoft.Json.Linq; using Squidex.Domain.Apps.Entities.Assets.Repositories; using Squidex.Domain.Apps.Entities.Assets.State; using Squidex.Domain.Apps.Entities.Backup; @@ -84,29 +85,23 @@ namespace Squidex.Domain.Apps.Entities.Assets await RebuildManyAsync(assetIds, id => RebuildAsync(id, (e, s) => s.Apply(e))); } - private Task RestoreTagsAsync(Guid appId, BackupReader reader) + private async Task RestoreTagsAsync(Guid appId, BackupReader reader) { - return reader.ReadAttachmentAsync(TagsFile, stream => - { - var tags = stream.DeserializeAsJson(); + var tags = await reader.ReadJsonAttachmentAsync(TagsFile); - return tagService.RebuildTagsAsync(appId, TagGroups.Assets, tags); - }); + await tagService.RebuildTagsAsync(appId, TagGroups.Assets, tags.ToObject()); } - private Task BackupTagsAsync(Guid appId, BackupWriter writer) + private async Task BackupTagsAsync(Guid appId, BackupWriter writer) { - return writer.WriteAttachmentAsync(TagsFile, async stream => - { - var tags = await tagService.GetExportableTagsAsync(appId, TagGroups.Assets); + var tags = await tagService.GetExportableTagsAsync(appId, TagGroups.Assets); - stream.SerializeAsJson(tags); - }); + await writer.WriteJsonAsync(TagsFile, JObject.FromObject(tags)); } private Task WriteAssetAsync(Guid assetId, long fileVersion, BackupWriter writer) { - return writer.WriteAttachmentAsync(GetName(assetId, fileVersion), stream => + return writer.WriteBlobAsync(GetName(assetId, fileVersion), stream => { return assetStore.DownloadAsync(assetId.ToString(), fileVersion, null, stream); }); @@ -116,7 +111,7 @@ namespace Squidex.Domain.Apps.Entities.Assets { assetIds.Add(assetId); - return reader.ReadAttachmentAsync(GetName(assetId, fileVersion), async stream => + return reader.ReadBlobAsync(GetName(reader.OldGuid(assetId), fileVersion), async stream => { try { diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs index 8970cf26f..cc49fe3e5 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs @@ -9,14 +9,19 @@ using System; using System.IO; using System.IO.Compression; using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Squidex.Domain.Apps.Entities.Backup.Archive; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Backup { public sealed class BackupReader : DisposableObjectBase { + private static readonly JsonSerializer Serializer = new JsonSerializer(); + private readonly GuidMapper guidMapper = new GuidMapper(); private readonly ZipArchive archive; private int readEvents; private int readAttachments; @@ -44,7 +49,43 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public async Task ReadAttachmentAsync(string name, Func handler) + public Guid OldGuid(Guid newId) + { + return guidMapper.OldGuid(newId); + } + + public async Task ReadJsonAttachmentAsync(string name) + { + Guard.NotNullOrEmpty(name, nameof(name)); + + var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name)); + + if (attachmentEntry == null) + { + throw new FileNotFoundException("Cannot find attachment.", name); + } + + JToken result; + + using (var stream = attachmentEntry.Open()) + { + using (var textReader = new StreamReader(stream)) + { + using (var jsonReader = new JsonTextReader(textReader)) + { + result = await JToken.ReadFromAsync(jsonReader); + + guidMapper.NewGuids(result); + } + } + } + + readAttachments++; + + return result; + } + + public async Task ReadBlobAsync(string name, Func handler) { Guard.NotNullOrEmpty(name, nameof(name)); Guard.NotNull(handler, nameof(handler)); @@ -64,9 +105,10 @@ namespace Squidex.Domain.Apps.Entities.Backup readAttachments++; } - public async Task ReadEventsAsync(Func handler) + public async Task ReadEventsAsync(IStreamNameResolver streamNameResolver, Func handler) { Guard.NotNull(handler, nameof(handler)); + Guard.NotNull(streamNameResolver, nameof(streamNameResolver)); while (true) { @@ -79,9 +121,25 @@ namespace Squidex.Domain.Apps.Entities.Backup using (var stream = eventEntry.Open()) { - var storedEvent = stream.DeserializeAsJson(); + using (var textReader = new StreamReader(stream)) + { + using (var jsonReader = new JsonTextReader(textReader)) + { + var storedEvent = Serializer.Deserialize(jsonReader); + + storedEvent.Data.Payload = guidMapper.NewGuids(storedEvent.Data.Payload); + storedEvent.Data.Metadata = guidMapper.NewGuids(storedEvent.Data.Metadata); + + var streamName = streamNameResolver.WithNewId(storedEvent.StreamName, guidMapper.NewGuidString); + + storedEvent = new StoredEvent(streamName, + storedEvent.EventPosition, + storedEvent.EventStreamNumber, + storedEvent.Data); - await handler(storedEvent); + await handler(storedEvent); + } + } } readEvents++; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs deleted file mode 100644 index b638f9da4..000000000 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupSerializer.cs +++ /dev/null @@ -1,41 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.IO; -using Newtonsoft.Json; - -namespace Squidex.Domain.Apps.Entities.Backup -{ - public static class BackupSerializer - { - private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(); - - public static void SerializeAsJson(this Stream stream, T value) - { - using (var writer = new StreamWriter(stream)) - { - JsonSerializer.Serialize(writer, value); - } - } - - public static T DeserializeAsJson(this Stream stream) - { - using (var reader = new StreamReader(stream)) - { - return (T)JsonSerializer.Deserialize(reader, typeof(T)); - } - } - - public static void DeserializeAsJson(this Stream stream, T result) - { - using (var reader = new StreamReader(stream)) - { - JsonSerializer.Populate(reader, result); - } - } - } -} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs index 49c45f94a..920ae4641 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs @@ -9,6 +9,8 @@ using System; using System.IO; using System.IO.Compression; using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Squidex.Domain.Apps.Entities.Backup.Archive; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; @@ -17,6 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { public sealed class BackupWriter : DisposableObjectBase { + private static readonly JsonSerializer Serializer = new JsonSerializer(); private readonly ZipArchive archive; private int writtenEvents; private int writtenAttachments; @@ -31,9 +34,9 @@ namespace Squidex.Domain.Apps.Entities.Backup get { return writtenAttachments; } } - public BackupWriter(Stream stream) + public BackupWriter(Stream stream, bool keepOpen = false) { - archive = new ZipArchive(stream, ZipArchiveMode.Create, false); + archive = new ZipArchive(stream, ZipArchiveMode.Create, keepOpen); } protected override void DisposeObject(bool disposing) @@ -44,7 +47,27 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public async Task WriteAttachmentAsync(string name, Func handler) + public async Task WriteJsonAsync(string name, JToken value) + { + Guard.NotNullOrEmpty(name, nameof(name)); + + var attachmentEntry = archive.CreateEntry(ArchiveHelper.GetAttachmentPath(name)); + + using (var stream = attachmentEntry.Open()) + { + using (var textWriter = new StreamWriter(stream)) + { + using (var jsonWriter = new JsonTextWriter(textWriter)) + { + await value.WriteToAsync(jsonWriter); + } + } + } + + writtenAttachments++; + } + + public async Task WriteBlobAsync(string name, Func handler) { Guard.NotNullOrEmpty(name, nameof(name)); Guard.NotNull(handler, nameof(handler)); @@ -67,7 +90,13 @@ namespace Squidex.Domain.Apps.Entities.Backup using (var stream = eventEntry.Open()) { - stream.SerializeAsJson(storedEvent); + using (var textWriter = new StreamWriter(stream)) + { + using (var jsonWriter = new JsonTextWriter(textWriter)) + { + Serializer.Serialize(jsonWriter, storedEvent); + } + } } writtenEvents++; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs b/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs index 7c97f4cd5..c0f7ac827 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs @@ -12,71 +12,159 @@ using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Backup { - public static class GuidMapper + public sealed class GuidMapper { private static readonly int GuidLength = Guid.Empty.ToString().Length; + private readonly List<(JObject Source, string NewKey, string OldKey)> mappings = new List<(JObject Source, string NewKey, string OldKey)>(); + private readonly Dictionary oldToNewGuid = new Dictionary(); + private readonly Dictionary newToOldGuid = new Dictionary(); - public static void GenerateNewGuid(JToken jToken, Dictionary guids) + public Guid NewGuid(Guid oldGuid) { - if (jToken.Type == JTokenType.Object) + return oldToNewGuid.GetOrDefault(oldGuid); + } + + public Guid OldGuid(Guid newGuid) + { + return newToOldGuid.GetOrDefault(newGuid); + } + + public string NewGuidString(string key) + { + if (Guid.TryParse(key, out var guid)) { - GenerateNewGuid((JObject)jToken, guids); + return GenerateNewGuid(guid).ToString(); } + + return null; } - private static void GenerateNewGuid(JObject jObject, Dictionary guids) + public JToken NewGuids(JToken jToken) { - foreach (var kvp in jObject) + var result = NewGuidsCore(jToken); + + if (mappings.Count > 0) { - switch (kvp.Value.Type) + foreach (var mapping in mappings) { - case JTokenType.String: - ReplaceGuidString(jObject, guids, kvp); - break; - case JTokenType.Guid: - ReplaceGuid(jObject, guids, kvp); - break; - case JTokenType.Object: - GenerateNewGuid((JObject)kvp.Value, guids); - break; + if (mapping.Source.TryGetValue(mapping.OldKey, out var value)) + { + mapping.Source.Remove(mapping.OldKey); + mapping.Source[mapping.NewKey] = value; + } } + + mappings.Clear(); + } + + return result; + } + + private JToken NewGuidsCore(JToken jToken) + { + switch (jToken.Type) + { + case JTokenType.String: + if (TryConvertString(jToken.ToString(), out var result)) + { + return result; + } + + break; + case JTokenType.Guid: + return GenerateNewGuid((Guid)jToken); + case JTokenType.Object: + NewGuidsCore((JObject)jToken); + break; + case JTokenType.Array: + NewGuidsCore((JArray)jToken); + break; + } + + return jToken; + } + + private void NewGuidsCore(JArray jArray) + { + for (var i = 0; i < jArray.Count; i++) + { + jArray[i] = NewGuidsCore(jArray[i]); } } - private static void ReplaceGuidString(JObject jObject, Dictionary guids, KeyValuePair kvp) + private void NewGuidsCore(JObject jObject) { - var value = kvp.Value.ToString(); + foreach (var jProperty in jObject.Properties()) + { + var newValue = NewGuidsCore(jProperty.Value); + + if (!ReferenceEquals(newValue, jProperty.Value)) + { + jProperty.Value = newValue; + } + + if (TryConvertString(jProperty.Name, out var newKey)) + { + mappings.Add((jObject, newKey, jProperty.Name)); + } + } + } + + private bool TryConvertString(string value, out string result) + { + return TryGenerateNewGuidString(value, out result) || TryGenerateNewNamedId(value, out result); + } + + private bool TryGenerateNewGuidString(string value, out string result) + { + result = null; if (value.Length == GuidLength) { if (Guid.TryParse(value, out var guid)) { - var newGuid = guids.GetOrAdd(guid, GuidGenerator); + var newGuid = GenerateNewGuid(guid); - jObject.Property(kvp.Key).Value = newGuid.ToString(); + result = newGuid.ToString(); + + return true; } } - else if (value.Length > GuidLength && value[GuidLength] == ',') + + return false; + } + + private bool TryGenerateNewNamedId(string value, out string result) + { + result = null; + + if (value.Length > GuidLength && value[GuidLength] == ',') { if (Guid.TryParse(value.Substring(0, GuidLength), out var guid)) { - var newGuid = guids.GetOrAdd(guid, GuidGenerator); + var newGuid = GenerateNewGuid(guid); - jObject.Property(kvp.Key).Value = newGuid + value.Substring(GuidLength); + result = newGuid + value.Substring(GuidLength); + + return true; } } + + return false; } - private static void ReplaceGuid(JObject jObject, Dictionary guids, KeyValuePair kvp) + private Guid GenerateNewGuid(Guid oldGuid) { - var newGuid = guids.GetOrAdd((Guid)kvp.Value, GuidGenerator); - - jObject.Property(kvp.Key).Value = newGuid; + return oldToNewGuid.GetOrAdd(oldGuid, GuidGenerator); } - private static Guid GuidGenerator(Guid key) + private Guid GuidGenerator(Guid oldGuid) { - return Guid.NewGuid(); + var newGuid = Guid.NewGuid(); + + newToOldGuid[newGuid] = oldGuid; + + return newGuid; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs index f38df5629..733e13e94 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs @@ -14,7 +14,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { public interface IRestoreGrain : IGrainWithStringKey { - Task RestoreAsync(Uri url); + Task RestoreAsync(string appName, Uri url); Task> GetJobAsync(); } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index 6200c7bf5..45f7e29af 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -34,6 +34,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private readonly IEventDataFormatter eventDataFormatter; private readonly IGrainFactory grainFactory; private readonly ISemanticLog log; + private readonly IStreamNameResolver streamNameResolver; private readonly IStore store; private RefToken actor; private RestoreState state = new RestoreState(); @@ -53,6 +54,7 @@ namespace Squidex.Domain.Apps.Entities.Backup IGrainFactory grainFactory, IEnumerable handlers, ISemanticLog log, + IStreamNameResolver streamNameResolver, IStore store) { Guard.NotNull(assetStore, nameof(assetStore)); @@ -63,6 +65,7 @@ namespace Squidex.Domain.Apps.Entities.Backup Guard.NotNull(grainFactory, nameof(grainFactory)); Guard.NotNull(handlers, nameof(handlers)); Guard.NotNull(store, nameof(store)); + Guard.NotNull(streamNameResolver, nameof(streamNameResolver)); Guard.NotNull(log, nameof(log)); this.assetStore = assetStore; @@ -73,6 +76,7 @@ namespace Squidex.Domain.Apps.Entities.Backup this.grainFactory = grainFactory; this.handlers = handlers; this.store = store; + this.streamNameResolver = streamNameResolver; this.log = log; } @@ -105,9 +109,10 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public Task RestoreAsync(Uri url) + public Task RestoreAsync(string appName, Uri url) { Guard.NotNull(url, nameof(url)); + Guard.NotNullOrEmpty(appName, nameof(appName)); if (CurrentJob?.Status == JobStatus.Started) { @@ -117,6 +122,7 @@ namespace Squidex.Domain.Apps.Entities.Backup state.Job = new RestoreStateJob { Id = Guid.NewGuid(), + AppName = appName, Started = clock.GetCurrentInstant(), Status = JobStatus.Started, Url = url @@ -255,7 +261,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private async Task ReadEventsAsync(BackupReader reader) { - await reader.ReadEventsAsync(async (storedEvent) => + await reader.ReadEventsAsync(streamNameResolver, async (storedEvent) => { var @event = eventDataFormatter.Parse(storedEvent.Data); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs index 4140fcae2..fb0694a61 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs @@ -14,6 +14,9 @@ namespace Squidex.Domain.Apps.Entities.Backup.State { public sealed class RestoreStateJob : IRestoreJob { + [JsonProperty] + public string AppName { get; set; } + [JsonProperty] public Guid Id { get; set; } diff --git a/src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs b/src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs deleted file mode 100644 index 4c9d97206..000000000 --- a/src/Squidex.Domain.Apps.Entities/ICleanableAppGrain.cs +++ /dev/null @@ -1,17 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.Threading.Tasks; -using Orleans; - -namespace Squidex.Domain.Apps.Entities -{ - public interface ICleanableAppGrain : IGrainWithGuidKey - { - Task ClearAsync(); - } -} diff --git a/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs b/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs index b2c00d6b7..2351cd68d 100644 --- a/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs +++ b/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs @@ -15,6 +15,9 @@ namespace Squidex.Infrastructure.States public string GetStreamName(Type aggregateType, string id) { + Guard.NotNullOrEmpty(id, nameof(id)); + Guard.NotNull(aggregateType, nameof(aggregateType)); + var typeName = char.ToLower(aggregateType.Name[0]) + aggregateType.Name.Substring(1); foreach (var suffix in Suffixes) @@ -29,5 +32,25 @@ namespace Squidex.Infrastructure.States return $"{typeName}-{id}"; } + + public string WithNewId(string streamName, Func idGenerator) + { + Guard.NotNullOrEmpty(streamName, nameof(streamName)); + Guard.NotNull(idGenerator, nameof(idGenerator)); + + var positionOfDash = streamName.LastIndexOf('-'); + + if (positionOfDash >= 0) + { + var newId = idGenerator(streamName.Substring(positionOfDash + 1)); + + if (!string.IsNullOrWhiteSpace(newId)) + { + streamName = $"{streamName.Substring(0, positionOfDash)}-{newId}"; + } + } + + return streamName; + } } } diff --git a/src/Squidex.Infrastructure/States/IStreamNameResolver.cs b/src/Squidex.Infrastructure/States/IStreamNameResolver.cs index a8d13034c..02b15f2fb 100644 --- a/src/Squidex.Infrastructure/States/IStreamNameResolver.cs +++ b/src/Squidex.Infrastructure/States/IStreamNameResolver.cs @@ -12,5 +12,7 @@ namespace Squidex.Infrastructure.States public interface IStreamNameResolver { string GetStreamName(Type aggregateType, string id); + + string WithNewId(string streamName, Func idGenerator); } } diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs index 50dd8ec55..f1b4528cc 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs @@ -12,6 +12,13 @@ namespace Squidex.Areas.Api.Controllers.Backups.Models { public sealed class RestoreRequest { + /// + /// The name of the app. + /// + [Required] + [RegularExpression("^[a-z0-9]+(\\-[a-z0-9]+)*$")] + public string Name { get; set; } + /// /// The url to the restore file. /// diff --git a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs index 4965bfd9e..553912221 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs @@ -63,7 +63,7 @@ namespace Squidex.Areas.Api.Controllers.Backups { var restoreGrain = grainFactory.GetGrain(User.OpenIdSubject()); - await restoreGrain.RestoreAsync(request.Url); + await restoreGrain.RestoreAsync(request.Name, request.Url); return NoContent(); } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs index fb80ca92d..4db3961c5 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs @@ -5,11 +5,15 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; +using FakeItEasy; using FluentAssertions; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.Tasks; using Xunit; @@ -17,6 +21,14 @@ namespace Squidex.Domain.Apps.Entities.Backup { public class BackupReaderWriterTests { + private readonly IStreamNameResolver streamNameResolver = A.Fake(); + + public BackupReaderWriterTests() + { + A.CallTo(() => streamNameResolver.WithNewId(A.Ignored, A>.Ignored)) + .ReturnsLazily(new Func, string>((stream, idGenerator) => stream + "^2")); + } + [Fact] public async Task Should_write_and_read_events() { @@ -24,20 +36,26 @@ namespace Squidex.Domain.Apps.Entities.Backup var sourceEvents = new List(); - using (var writer = new BackupWriter(stream)) + using (var writer = new BackupWriter(stream, true)) { for (var i = 0; i < 1000; i++) { var eventData = new EventData { Type = i.ToString(), Metadata = i, Payload = i }; var eventStored = new StoredEvent("S", "1", 2, eventData); - if (i % 10 == 0) + if (i % 17 == 0) { - await writer.WriteAttachmentAsync(eventData.Type, innerStream => + await writer.WriteBlobAsync(eventData.Type, innerStream => { - return innerStream.WriteAsync(new byte[] { (byte)i }, 0, 1); + innerStream.WriteByte((byte)i); + + return TaskHelper.Done; }); } + else if (i % 37 == 0) + { + await writer.WriteJsonAsync(eventData.Type, $"JSON_{i}"); + } writer.WriteEvent(eventStored); @@ -51,13 +69,13 @@ namespace Squidex.Domain.Apps.Entities.Backup using (var reader = new BackupReader(stream)) { - await reader.ReadEventsAsync(async @event => + await reader.ReadEventsAsync(streamNameResolver, async @event => { var i = int.Parse(@event.Data.Type); - if (i % 10 == 0) + if (i % 17 == 0) { - await reader.ReadAttachmentAsync(@event.Data.Type, innerStream => + await reader.ReadBlobAsync(@event.Data.Type, innerStream => { var b = innerStream.ReadByte(); @@ -66,12 +84,25 @@ namespace Squidex.Domain.Apps.Entities.Backup return TaskHelper.Done; }); } + else if (i % 37 == 0) + { + var j = await reader.ReadJsonAttachmentAsync(@event.Data.Type); + + Assert.Equal($"JSON_{i}", j.ToString()); + } readEvents.Add(@event); }); } - readEvents.Should().BeEquivalentTo(sourceEvents); + var sourceEventsWithNewStreamName = + sourceEvents.Select(x => + new StoredEvent(streamNameResolver.WithNewId(x.StreamName, null), + x.EventPosition, + x.EventStreamNumber, + x.Data)).ToList(); + + readEvents.Should().BeEquivalentTo(sourceEventsWithNewStreamName); } } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs index a482cde0a..fdab2731a 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Backup/GuidMapperTests.cs @@ -6,7 +6,6 @@ // ========================================================================== using System; -using System.Collections.Generic; using Newtonsoft.Json.Linq; using Xunit; @@ -14,14 +13,100 @@ namespace Squidex.Domain.Apps.Entities.Backup { public class GuidMapperTests { + private readonly Guid id1 = Guid.NewGuid(); + private readonly Guid id2 = Guid.NewGuid(); + private readonly GuidMapper map = new GuidMapper(); + + [Fact] + public void Should_map_guid_string_if_valid() + { + var result = map.NewGuidString(id1.ToString()); + + Assert.Equal(map.NewGuid(id1).ToString(), result); + } + + [Fact] + public void Should_return_null_if_mapping_invalid_guid_string() + { + var result = map.NewGuidString("invalid"); + + Assert.Null(result); + } + + [Fact] + public void Should_return_null_if_mapping_null_guid_string() + { + var result = map.NewGuidString(null); + + Assert.Null(result); + } + [Fact] public void Should_map_guid() { - var m = new Dictionary(); + var result = map.NewGuids(id1); - var id1 = Guid.NewGuid(); - var id2 = Guid.NewGuid(); + Assert.Equal(map.NewGuid(id1), result.Value()); + } + + [Fact] + public void Should_return_old_guid() + { + var newGuid = map.NewGuids(id1).Value(); + Assert.Equal(id1, map.OldGuid(newGuid)); + } + + [Fact] + public void Should_map_guid_string() + { + var result = map.NewGuids(id1.ToString()); + + Assert.Equal(map.NewGuid(id1).ToString(), result.Value()); + } + + [Fact] + public void Should_map_named_id() + { + var result = map.NewGuids($"{id1},name"); + + Assert.Equal($"{map.NewGuid(id1)},name", result.Value()); + } + + [Fact] + public void Should_map_array_with_guid() + { + var obj = + new JObject( + new JProperty("k", + new JArray(id1, id1, id2))); + + map.NewGuids(obj); + + Assert.Equal(map.NewGuid(id1), obj["k"][0].Value()); + Assert.Equal(map.NewGuid(id1), obj["k"][1].Value()); + Assert.Equal(map.NewGuid(id2), obj["k"][2].Value()); + } + + [Fact] + public void Should_map_objects_with_guid_keys() + { + var obj = + new JObject( + new JProperty("k", + new JObject( + new JProperty(id1.ToString(), id1), + new JProperty(id2.ToString(), id2)))); + + map.NewGuids(obj); + + Assert.Equal(map.NewGuid(id1), obj["k"].Value(map.NewGuid(id1).ToString())); + Assert.Equal(map.NewGuid(id2), obj["k"].Value(map.NewGuid(id2).ToString())); + } + + [Fact] + public void Should_map_objects_with_guid() + { var obj = new JObject( new JProperty("k", @@ -30,21 +115,16 @@ namespace Squidex.Domain.Apps.Entities.Backup new JProperty("v2", id1), new JProperty("v3", id2)))); - GuidMapper.GenerateNewGuid(obj, m); + map.NewGuids(obj); - Assert.Equal(m[id1], obj["k"].Value("v1")); - Assert.Equal(m[id1], obj["k"].Value("v2")); - Assert.Equal(m[id2], obj["k"].Value("v3")); + Assert.Equal(map.NewGuid(id1), obj["k"].Value("v1")); + Assert.Equal(map.NewGuid(id1), obj["k"].Value("v2")); + Assert.Equal(map.NewGuid(id2), obj["k"].Value("v3")); } [Fact] - public void Should_map_guid_string() + public void Should_map_objects_with_guid_string() { - var m = new Dictionary(); - - var id1 = Guid.NewGuid(); - var id2 = Guid.NewGuid(); - var obj = new JObject( new JProperty("k", @@ -53,21 +133,16 @@ namespace Squidex.Domain.Apps.Entities.Backup new JProperty("v2", id1.ToString()), new JProperty("v3", id2.ToString())))); - GuidMapper.GenerateNewGuid(obj, m); + map.NewGuids(obj); - Assert.Equal(m[id1].ToString(), obj["k"].Value("v1")); - Assert.Equal(m[id1].ToString(), obj["k"].Value("v2")); - Assert.Equal(m[id2].ToString(), obj["k"].Value("v3")); + Assert.Equal(map.NewGuid(id1).ToString(), obj["k"].Value("v1")); + Assert.Equal(map.NewGuid(id1).ToString(), obj["k"].Value("v2")); + Assert.Equal(map.NewGuid(id2).ToString(), obj["k"].Value("v3")); } [Fact] - public void Should_map_named_id() + public void Should_map_objects_with_named_id() { - var m = new Dictionary(); - - var id1 = Guid.NewGuid(); - var id2 = Guid.NewGuid(); - var obj = new JObject( new JProperty("k", @@ -76,11 +151,11 @@ namespace Squidex.Domain.Apps.Entities.Backup new JProperty("v2", $"{id1},v2"), new JProperty("v3", $"{id2},v3")))); - GuidMapper.GenerateNewGuid(obj, m); + map.NewGuids(obj); - Assert.Equal($"{m[id1].ToString()},v1", obj["k"].Value("v1")); - Assert.Equal($"{m[id1].ToString()},v2", obj["k"].Value("v2")); - Assert.Equal($"{m[id2].ToString()},v3", obj["k"].Value("v3")); + Assert.Equal($"{map.NewGuid(id1).ToString()},v1", obj["k"].Value("v1")); + Assert.Equal($"{map.NewGuid(id1).ToString()},v2", obj["k"].Value("v2")); + Assert.Equal($"{map.NewGuid(id2).ToString()},v3", obj["k"].Value("v3")); } } } diff --git a/tests/Squidex.Infrastructure.Tests/States/DefaultStreamNameResolverTests.cs b/tests/Squidex.Infrastructure.Tests/States/DefaultStreamNameResolverTests.cs index 26f36ef21..d643ee9d1 100644 --- a/tests/Squidex.Infrastructure.Tests/States/DefaultStreamNameResolverTests.cs +++ b/tests/Squidex.Infrastructure.Tests/States/DefaultStreamNameResolverTests.cs @@ -43,5 +43,35 @@ namespace Squidex.Infrastructure.States Assert.Equal($"myUser-{id}", name); } + + [Fact] + public void Should_calculate_new_stream_if_valid() + { + var oldStream = "myUser-123"; + + var newStream = sut.WithNewId(oldStream, x => "456"); + + Assert.Equal("myUser-456", newStream); + } + + [Fact] + public void Should_return_old_stream_if_format_not_valid() + { + var oldStream = "myUser|123"; + + var newStream = sut.WithNewId(oldStream, x => "456"); + + Assert.Equal(oldStream, newStream); + } + + [Fact] + public void Should_return_old_stream_if_new_id_not_valid() + { + var oldStream = "myUser-123"; + + var newStream = sut.WithNewId(oldStream, x => null); + + Assert.Equal(oldStream, newStream); + } } } From 5d5ce884c96a6e8032438f7ddbbda0828fe5fbe3 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sun, 19 Aug 2018 13:13:28 +0200 Subject: [PATCH 21/23] app name --- .../Backup/IRestoreGrain.cs | 2 +- .../Backup/RestoreGrain.cs | 18 +++++++++++++++++- .../Backup/State/RestoreStateJob.cs | 3 +++ .../Backups/Models/RestoreRequest.cs | 3 +++ .../Controllers/Backups/RestoreController.cs | 2 +- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs index f38df5629..6ee281401 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs @@ -14,7 +14,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { public interface IRestoreGrain : IGrainWithStringKey { - Task RestoreAsync(Uri url); + Task RestoreAsync(Uri url, string newAppName = null); Task> GetJobAsync(); } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index 6200c7bf5..70a7d5a50 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -105,10 +105,15 @@ namespace Squidex.Domain.Apps.Entities.Backup } } - public Task RestoreAsync(Uri url) + public Task RestoreAsync(Uri url, string newAppName) { Guard.NotNull(url, nameof(url)); + if (newAppName != null) + { + Guard.ValidSlug(newAppName, nameof(newAppName)); + } + if (CurrentJob?.Status == JobStatus.Started) { throw new DomainException("A restore operation is already running."); @@ -117,6 +122,7 @@ namespace Squidex.Domain.Apps.Entities.Backup state.Job = new RestoreStateJob { Id = Guid.NewGuid(), + NewAppName = newAppName, Started = clock.GetCurrentInstant(), Status = JobStatus.Started, Url = url @@ -267,6 +273,16 @@ namespace Squidex.Domain.Apps.Entities.Backup if (@event.Payload is AppCreated appCreated) { CurrentJob.AppId = appCreated.AppId.Id; + + if (!string.IsNullOrWhiteSpace(CurrentJob.NewAppName)) + { + appCreated.Name = CurrentJob.NewAppName; + } + } + + if (@event.Payload is AppEvent appEvent && !string.IsNullOrWhiteSpace(CurrentJob.NewAppName)) + { + appEvent.AppId = new NamedId(appEvent.AppId.Id, CurrentJob.NewAppName); } foreach (var handler in handlers) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs index 4140fcae2..e601232a8 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs @@ -23,6 +23,9 @@ namespace Squidex.Domain.Apps.Entities.Backup.State [JsonProperty] public Uri Url { get; set; } + [JsonProperty] + public string NewAppName { get; set; } + [JsonProperty] public Instant Started { get; set; } diff --git a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs index 50dd8ec55..3b4154f0c 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs @@ -17,5 +17,8 @@ namespace Squidex.Areas.Api.Controllers.Backups.Models /// [Required] public Uri Url { get; set; } + + [RegularExpression("^[a-z0-9]+(\\-[a-z0-9]+)*$")] + public string Name { get; set; } } } diff --git a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs index 4965bfd9e..2162860a5 100644 --- a/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs +++ b/src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs @@ -63,7 +63,7 @@ namespace Squidex.Areas.Api.Controllers.Backups { var restoreGrain = grainFactory.GetGrain(User.OpenIdSubject()); - await restoreGrain.RestoreAsync(request.Url); + await restoreGrain.RestoreAsync(request.Url, request.Name); return NoContent(); } From e052364595fdbfefd35a0745982cc28bf05c5939 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sun, 19 Aug 2018 14:51:30 +0200 Subject: [PATCH 22/23] Bugfixes for restore --- src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs | 3 +++ .../Backup/BackupGrain.cs | 2 +- .../States/DefaultStreamNameResolver.cs | 2 +- .../pages/restore/restore-page.component.html | 3 +++ .../pages/restore/restore-page.component.ts | 2 +- .../app/shared/services/backups.service.spec.ts | 5 +++-- src/Squidex/app/shared/services/backups.service.ts | 12 ++++++++++-- src/Squidex/app/shared/state/backups.forms.ts | 8 +++++++- 8 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs index c54ae17a5..eed1cdf70 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Newtonsoft.Json.Linq; using Orleans; using Squidex.Domain.Apps.Entities.Apps.Indexes; +using Squidex.Domain.Apps.Entities.Apps.State; using Squidex.Domain.Apps.Entities.Backup; using Squidex.Domain.Apps.Events; using Squidex.Domain.Apps.Events.Apps; @@ -166,6 +167,8 @@ namespace Squidex.Domain.Apps.Entities.Apps public override async Task CompleteRestoreAsync(Guid appId, BackupReader reader) { + await RebuildAsync(appId, (e, s) => s.Apply(e)); + await grainFactory.GetGrain(SingleGrain.Id).AddAppAsync(appCreated.AppId.Id, appCreated.AppId.Name); foreach (var user in activeUsers) diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index b1f87e841..21067ce66 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -139,7 +139,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { using (var stream = await backupArchiveLocation.OpenStreamAsync(job.Id)) { - using (var writer = new BackupWriter(stream)) + using (var writer = new BackupWriter(stream, true)) { await eventStore.QueryAsync(async storedEvent => { diff --git a/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs b/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs index 2351cd68d..582011a20 100644 --- a/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs +++ b/src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs @@ -38,7 +38,7 @@ namespace Squidex.Infrastructure.States Guard.NotNullOrEmpty(streamName, nameof(streamName)); Guard.NotNull(idGenerator, nameof(idGenerator)); - var positionOfDash = streamName.LastIndexOf('-'); + var positionOfDash = streamName.IndexOf('-'); if (positionOfDash >= 0) { diff --git a/src/Squidex/app/features/administration/pages/restore/restore-page.component.html b/src/Squidex/app/features/administration/pages/restore/restore-page.component.html index 82dbb493b..9ddcbc968 100644 --- a/src/Squidex/app/features/administration/pages/restore/restore-page.component.html +++ b/src/Squidex/app/features/administration/pages/restore/restore-page.component.html @@ -55,6 +55,9 @@
    +
    + +
    diff --git a/src/Squidex/app/features/administration/pages/restore/restore-page.component.ts b/src/Squidex/app/features/administration/pages/restore/restore-page.component.ts index bc3883734..fa8977ccf 100644 --- a/src/Squidex/app/features/administration/pages/restore/restore-page.component.ts +++ b/src/Squidex/app/features/administration/pages/restore/restore-page.component.ts @@ -57,7 +57,7 @@ export class RestorePageComponent implements OnDestroy, OnInit { if (value) { this.restoreForm.submitCompleted({}); - this.backupsService.postRestore(value.url) + this.backupsService.postRestore(value) .subscribe(() => { this.dialogs.notifyInfo('Restore started, it can take several minutes to complete.'); }, error => { diff --git a/src/Squidex/app/shared/services/backups.service.spec.ts b/src/Squidex/app/shared/services/backups.service.spec.ts index 22a51cb2d..c88b0f161 100644 --- a/src/Squidex/app/shared/services/backups.service.spec.ts +++ b/src/Squidex/app/shared/services/backups.service.spec.ts @@ -14,7 +14,8 @@ import { BackupDto, BackupsService, DateTime, - RestoreDto + RestoreDto, + StartRestoreDto } from './../'; describe('BackupsService', () => { @@ -169,7 +170,7 @@ describe('BackupsService', () => { it('should make post request to start restore', inject([BackupsService, HttpTestingController], (backupsService: BackupsService, httpMock: HttpTestingController) => { - backupsService.postRestore('http://url').subscribe(); + backupsService.postRestore(new StartRestoreDto('http://url')).subscribe(); const req = httpMock.expectOne('http://service/p/api/apps/restore'); diff --git a/src/Squidex/app/shared/services/backups.service.ts b/src/Squidex/app/shared/services/backups.service.ts index 676e968a9..68b8298a3 100644 --- a/src/Squidex/app/shared/services/backups.service.ts +++ b/src/Squidex/app/shared/services/backups.service.ts @@ -43,6 +43,14 @@ export class RestoreDto { } } +export class StartRestoreDto { + constructor( + public readonly url: string, + public readonly newAppName?: string + ) { + } +} + @Injectable() export class BackupsService { constructor( @@ -106,10 +114,10 @@ export class BackupsService { pretifyError('Failed to start backup.')); } - public postRestore(downloadUrl: string): Observable { + public postRestore(dto: StartRestoreDto): Observable { const url = this.apiUrl.buildUrl(`api/apps/restore`); - return this.http.post(url, { url: downloadUrl }).pipe( + return this.http.post(url, dto).pipe( tap(() => { this.analytics.trackEvent('Restore', 'Started'); }), diff --git a/src/Squidex/app/shared/state/backups.forms.ts b/src/Squidex/app/shared/state/backups.forms.ts index cbdb82d5b..475050a6c 100644 --- a/src/Squidex/app/shared/state/backups.forms.ts +++ b/src/Squidex/app/shared/state/backups.forms.ts @@ -8,7 +8,7 @@ import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { map, startWith } from 'rxjs/operators'; -import { Form } from '@app/framework'; +import { Form, ValidatorsEx } from '@app/framework'; export class RestoreForm extends Form { public hasNoUrl = @@ -16,6 +16,12 @@ export class RestoreForm extends Form { constructor(formBuilder: FormBuilder) { super(formBuilder.group({ + name: [null, + [ + Validators.maxLength(40), + ValidatorsEx.pattern('[a-z0-9]+(\-[a-z0-9]+)*', 'Name can contain lower case letters (a-z), numbers and dashes (not at the end).') + ] + ], url: [null, [ Validators.required From 3da4035bde16200b5bd910f5e9773cf5575c2361 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sun, 19 Aug 2018 15:54:40 +0200 Subject: [PATCH 23/23] Tests fixed --- .../Contents/GraphQL/Types/AllTypes.cs | 2 +- .../Tags/GrainTagServiceTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/AllTypes.cs b/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/AllTypes.cs index f9f25ad41..813d62b93 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/AllTypes.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/AllTypes.cs @@ -20,7 +20,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.GraphQL.Types public static readonly IGraphType Guid = new GuidGraphType(); - public static readonly IGraphType Date = new DateGraphType(); + public static readonly IGraphType Date = new DateTimeGraphType(); public static readonly IGraphType Json = new JsonGraphType(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs index c3667ee77..ec40dc92b 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Tags/GrainTagServiceTests.cs @@ -60,7 +60,7 @@ namespace Squidex.Domain.Apps.Entities.Tags { await sut.GetExportableTagsAsync(appId, TagGroups.Assets); - A.CallTo(() => grain.GetTagsAsync()) + A.CallTo(() => grain.GetExportableTagsAsync()) .MustHaveHappened(); }