diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs index 6cdbfecc6..189f08052 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs @@ -18,11 +18,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { public sealed partial class MongoAssetRepository : ISnapshotStore { - Task ISnapshotStore.ReadAllAsync(Func callback) - { - throw new NotSupportedException(); - } - public async Task<(AssetState Value, long Version)> ReadAsync(Guid key) { using (Profiler.TraceMethod()) @@ -52,5 +47,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets await Collection.ReplaceOneAsync(x => x.Id == key && x.Version == oldVersion, entity, Upsert); } } + + Task ISnapshotStore.ReadAllAsync(Func callback) + { + throw new NotSupportedException(); + } + + Task ISnapshotStore.RemoveAsync(Guid key) + { + throw new NotSupportedException(); + } } } diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs index ad6854d8b..eccc17915 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs @@ -117,5 +117,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents Filter.AnyNe(x => x.ReferencedIdsDeleted, id)), Update.AddToSet(x => x.ReferencedIdsDeleted, id)); } + + public Task RemoveAsync(Guid id) + { + return Collection.DeleteOneAsync(x => x.Id == id); + } } } diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs index 42a5050c9..ae894f6cf 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs @@ -58,6 +58,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return ids.Except(contentEntities.Select(x => Guid.Parse(x["_id"].AsString))).ToList(); } + public async Task> QueryIdsAsync(Guid appId) + { + var contentEntities = + await Collection.Find(x => x.IndexedAppId == appId).Only(x => x.Id) + .ToListAsync(); + + return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList(); + } + public Task QueryScheduledWithoutDataAsync(Instant now, Func callback) { return Collection.Find(x => x.ScheduledAt < now && x.IsDeleted != true) diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index 0d1f94f84..fb19628b3 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -99,6 +99,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } + public async Task> QueryIdsAsync(Guid appId) + { + using (Profiler.TraceMethod()) + { + return await contentsDraft.QueryIdsAsync(appId); + } + } + public async Task QueryScheduledWithoutDataAsync(Instant now, Func callback) { using (Profiler.TraceMethod()) diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs index 8393c28c3..cd6700635 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs @@ -83,6 +83,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return schema; } + Task ISnapshotStore.RemoveAsync(Guid key) + { + throw new NotSupportedException(); + } + Task ISnapshotStore.ReadAllAsync(Func callback) { throw new NotSupportedException(); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs new file mode 100644 index 000000000..a0db8d3fd --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/CleanerGrain.cs @@ -0,0 +1,147 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Orleans; +using Orleans.Concurrency; +using Orleans.Runtime; +using Squidex.Domain.Apps.Entities.Apps.State; +using Squidex.Domain.Apps.Entities.Rules; +using Squidex.Domain.Apps.Entities.Rules.State; +using Squidex.Domain.Apps.Entities.Schemas; +using Squidex.Domain.Apps.Entities.Schemas.State; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Orleans; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + [Reentrant] + public sealed class CleanerGrain : GrainOfGuid, IRemindable + { + private readonly IGrainFactory grainFactory; + private readonly IStore store; + private readonly IEventStore eventStore; + private readonly IEnumerable storages; + private IPersistence persistence; + private bool isCleaning; + private State state = new State(); + + [CollectionName("Index_AppsByName")] + public sealed class State + { + public HashSet Apps { get; set; } = new HashSet(); + + public HashSet PendingApps { get; set; } = new HashSet(); + } + + public CleanerGrain(IGrainFactory grainFactory, IEventStore eventStore, IStore store, IEnumerable storages) + { + Guard.NotNull(grainFactory, nameof(grainFactory)); + Guard.NotNull(store, nameof(store)); + Guard.NotNull(storages, nameof(storages)); + Guard.NotNull(eventStore, nameof(eventStore)); + + this.grainFactory = grainFactory; + + this.store = store; + this.storages = storages; + + this.eventStore = eventStore; + } + + public async override Task OnActivateAsync(Guid key) + { + await RegisterOrUpdateReminder("Default", TimeSpan.Zero, TimeSpan.FromMinutes(10)); + + persistence = store.WithSnapshots(key, s => + { + state = s; + }); + + await persistence.ReadAsync(); + + await CleanAsync(); + } + + public Task ReceiveReminder(string reminderName, TickStatus status) + { + return CleanAsync(); + } + + private async Task CleanAsync() + { + if (isCleaning) + { + return; + } + + isCleaning = true; + try + { + foreach (var appId in state.Apps.ToList()) + { + try + { + await CleanAsync(appId); + + state.Apps.Remove(appId); + } + catch (NotSupportedException) + { + state.Apps.Remove(appId); + + state.PendingApps.Add(appId); + } + finally + { + await persistence.WriteSnapshotAsync(state); + } + } + } + finally + { + isCleaning = false; + } + } + + private async Task CleanAsync(Guid appId) + { + await eventStore.DeleteManyAsync("AppId", appId); + + var ruleIds = await grainFactory.GetGrain(appId).GetRuleIdsAsync(); + + foreach (var ruleId in ruleIds) + { + await store.ClearSnapshotAsync(ruleId); + } + + var schemaIds = await grainFactory.GetGrain(appId).GetSchemaIdsAsync(); + + foreach (var schemaId in schemaIds) + { + await store.ClearSnapshotAsync(schemaId); + } + + foreach (var storage in storages) + { + await storage.ClearAsync(appId); + } + + await store.ClearSnapshotAsync(appId; + } + + private async Task DeleteAsync(Guid id) + { + await store.ClearSnapshotAsync(id); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/IAppStorage.cs b/src/Squidex.Domain.Apps.Entities/IAppStorage.cs new file mode 100644 index 000000000..16a051157 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/IAppStorage.cs @@ -0,0 +1,17 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; + +namespace Squidex.Domain.Apps.Entities +{ + public interface IAppStorage + { + Task ClearAsync(Guid appId); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs index dd169a045..a1a54459e 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs @@ -20,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Rules Task RebuildAsync(HashSet rules); + Task ClearAsync(); + Task> GetRuleIdsAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs index 6f9a49390..b3b87943d 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs @@ -69,5 +69,10 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes { return Task.FromResult(state.Rules.ToList()); } + + public Task ClearAsync() + { + return persistence.DeleteAsync(); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs index 38104d363..58b9f9e1f 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs @@ -18,6 +18,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas Task RemoveSchemaAsync(Guid schemaId); + Task ClearAsync(); + Task RebuildAsync(Dictionary schemas); Task GetSchemaIdAsync(string name); diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs index 9164936c9..cbec766f3 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs @@ -76,5 +76,10 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes { return Task.FromResult(state.Schemas.Values.ToList()); } + + public Task ClearAsync() + { + return persistence.DeleteAsync(); + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs index a37bf83cb..2947aebda 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs @@ -20,5 +20,7 @@ namespace Squidex.Domain.Apps.Entities.Tags Task> DenormalizeTagsAsync(HashSet ids); Task> GetTagsAsync(); + + Task ClearAsync(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs index 3517b9476..66e17e6cb 100644 --- a/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs @@ -147,5 +147,10 @@ namespace Squidex.Domain.Apps.Entities.Tags { return Task.FromResult(state.Tags.Values.ToDictionary(x => x.Name, x => x.Count)); } + + public Task ClearAsync() + { + return persistence.DeleteAsync(); + } } } diff --git a/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs b/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs index d21394747..50722a41d 100644 --- a/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs +++ b/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs @@ -118,6 +118,11 @@ namespace Squidex.Infrastructure.EventSourcing } } + public Task DeleteStreamAsync(string streamName) + { + return connection.DeleteStreamAsync(streamName, ExpectedVersion.Any); + } + public Task AppendAsync(Guid commitId, string streamName, ICollection events) { return AppendEventsInternalAsync(streamName, EtagVersion.Any, events); @@ -163,6 +168,11 @@ namespace Squidex.Infrastructure.EventSourcing } } + public Task DeleteManyAsync(string property, object value) + { + throw new NotSupportedException(); + } + private string GetStreamName(string streamName) { return $"{prefix}-{streamName}"; diff --git a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs index 9f7bc88f5..07861115d 100644 --- a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs +++ b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs @@ -124,8 +124,8 @@ namespace Squidex.Infrastructure.EventSourcing { var filters = new List>(); - AddPositionFilter(streamPosition, filters); - AddPropertyFitler(property, value, filters); + FilterByPosition(streamPosition, filters); + FilterByProperty(property, value, filters); return Filter.And(filters); } @@ -134,18 +134,18 @@ namespace Squidex.Infrastructure.EventSourcing { var filters = new List>(); - AddPositionFilter(streamPosition, filters); - AddStreamFilter(streamFilter, filters); + FilterByPosition(streamPosition, filters); + FilterByStream(streamFilter, filters); return Filter.And(filters); } - private static void AddPropertyFitler(string property, object value, List> filters) + private static void FilterByProperty(string property, object value, List> filters) { filters.Add(Filter.Eq(CreateIndexPath(property), value)); } - private static void AddStreamFilter(string streamFilter, List> filters) + private static void FilterByStream(string streamFilter, List> filters) { if (!string.IsNullOrWhiteSpace(streamFilter) && !string.Equals(streamFilter, ".*", StringComparison.OrdinalIgnoreCase)) { @@ -160,7 +160,7 @@ namespace Squidex.Infrastructure.EventSourcing } } - private static void AddPositionFilter(StreamPosition streamPosition, List> filters) + private static void FilterByPosition(StreamPosition streamPosition, List> filters) { if (streamPosition.IsEndOfCommit) { diff --git a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs index bbeb5c469..af4b0172a 100644 --- a/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs +++ b/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs @@ -20,6 +20,16 @@ namespace Squidex.Infrastructure.EventSourcing private const int MaxWriteAttempts = 20; private static readonly BsonTimestamp EmptyTimestamp = new BsonTimestamp(0); + public Task DeleteStreamAsync(string streamName) + { + return Collection.DeleteManyAsync(x => x.EventStream == streamName); + } + + public Task DeleteManyAsync(string property, object value) + { + return Collection.DeleteManyAsync(Filter.Eq(CreateIndexPath(property), value)); + } + public Task AppendAsync(Guid commitId, string streamName, ICollection events) { return AppendAsync(commitId, streamName, EtagVersion.Any, events); diff --git a/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs b/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs index 8fc6c3204..67e593b53 100644 --- a/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs @@ -68,5 +68,13 @@ namespace Squidex.Infrastructure.States await Collection.Find(new BsonDocument()).ForEachAsync(x => callback(x.Doc, x.Version)); } } + + public async Task RemoveAsync(TKey key) + { + using (Profiler.TraceMethod>()) + { + await Collection.DeleteOneAsync(x => x.Id.Equals(key)); + } + } } } diff --git a/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs b/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs index c33d86e5a..186b7310e 100644 --- a/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs +++ b/src/Squidex.Infrastructure/EventSourcing/IEventStore.cs @@ -26,6 +26,10 @@ namespace Squidex.Infrastructure.EventSourcing Task AppendAsync(Guid commitId, string streamName, long expectedVersion, ICollection events); + Task DeleteStreamAsync(string streamName); + + Task DeleteManyAsync(string property, object value); + IEventSubscription CreateSubscription(IEventSubscriber subscriber, string streamFilter, string position = null); } } diff --git a/src/Squidex.Infrastructure/States/IPersistence{TState}.cs b/src/Squidex.Infrastructure/States/IPersistence{TState}.cs index 2b467b302..c69e95926 100644 --- a/src/Squidex.Infrastructure/States/IPersistence{TState}.cs +++ b/src/Squidex.Infrastructure/States/IPersistence{TState}.cs @@ -15,6 +15,8 @@ namespace Squidex.Infrastructure.States { long Version { get; } + Task DeleteAsync(); + Task WriteEventsAsync(IEnumerable> @events); Task WriteSnapshotAsync(TState state); diff --git a/src/Squidex.Infrastructure/States/ISnapshotStore.cs b/src/Squidex.Infrastructure/States/ISnapshotStore.cs index 60b2ab5be..9431226f1 100644 --- a/src/Squidex.Infrastructure/States/ISnapshotStore.cs +++ b/src/Squidex.Infrastructure/States/ISnapshotStore.cs @@ -18,6 +18,8 @@ namespace Squidex.Infrastructure.States Task ClearAsync(); + Task RemoveAsync(TKey key); + Task ReadAllAsync(Func callback); } } diff --git a/src/Squidex.Infrastructure/States/IStore.cs b/src/Squidex.Infrastructure/States/IStore.cs index e0c65ba83..8dee5d085 100644 --- a/src/Squidex.Infrastructure/States/IStore.cs +++ b/src/Squidex.Infrastructure/States/IStore.cs @@ -22,5 +22,7 @@ namespace Squidex.Infrastructure.States ISnapshotStore GetSnapshotStore(); Task ClearSnapshotsAsync(); + + Task ClearSnapshotAsync(TKey key); } } diff --git a/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs b/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs index ea8b50d1e..5c6a22c3f 100644 --- a/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs +++ b/src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs @@ -175,6 +175,19 @@ namespace Squidex.Infrastructure.States UpdateVersion(); } + public async Task DeleteAsync() + { + if (UseEventSourcing()) + { + await eventStore.DeleteStreamAsync(GetStreamName()); + } + + if (UseSnapshots()) + { + await snapshotStore.RemoveAsync(ownerKey); + } + } + private EventData[] GetEventData(Envelope[] events, Guid commitId) { return @events.Select(x => eventDataFormatter.ToEventData(x, commitId, true)).ToArray(); diff --git a/src/Squidex.Infrastructure/States/Store.cs b/src/Squidex.Infrastructure/States/Store.cs index 0a3e02ff7..6f693c5a2 100644 --- a/src/Squidex.Infrastructure/States/Store.cs +++ b/src/Squidex.Infrastructure/States/Store.cs @@ -63,6 +63,11 @@ namespace Squidex.Infrastructure.States return GetSnapshotStore().ClearAsync(); } + public Task ClearSnapshotAsync(TKey key) + { + return GetSnapshotStore().RemoveAsync(key); + } + public ISnapshotStore GetSnapshotStore() { return (ISnapshotStore)services.GetService(typeof(ISnapshotStore));