From a706ad5757ec4cc7fba4d0231da6213d8827a16e Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 Mar 2018 14:19:09 +0100 Subject: [PATCH 1/7] Backup started. --- src/Squidex.Domain.Apps.Backup/BackupGrain.cs | 242 ++++++++++++++++++ .../EventStreamWriter.cs | 80 ++++++ .../IBackupArchiveLocation.cs | 13 + .../IBackupGrain.cs | 24 ++ src/Squidex.Domain.Apps.Backup/IBackupJob.cs | 25 ++ .../Squidex.Domain.Apps.Backup.csproj | 22 ++ .../State/BackupState.cs | 18 ++ .../State/BackupStateJob.cs | 31 +++ .../Apps/AppGrain.cs | 2 +- .../Assets/AssetCommandMiddleware.cs | 12 +- .../Backup/BackupGrain.cs | 242 ++++++++++++++++++ .../Backup/EventStreamWriter.cs | 79 ++++++ .../Backup/IBackupArchiveLocation.cs | 20 ++ .../Backup/IBackupGrain.cs | 24 ++ .../Backup/IBackupJob.cs | 23 ++ .../Backup/State/BackupState.cs | 18 ++ .../Backup/State/BackupStateJob.cs | 28 ++ .../Backup/TempFolderBackupArchiveLocation.cs | 44 ++++ .../Contents/ContentGrain.cs | 2 +- .../Contents/IContentScheduleItem.cs | 22 ++ .../State/ContentStateScheduleItem.cs | 26 ++ .../Rules/RuleGrain.cs | 2 +- .../Schemas/SchemaGrain.cs | 2 +- .../SquidexDomainObjectGrain.cs | 31 +++ .../SquidexEntities.cs | 2 +- .../Contents/ContentScheduleItemRemoved.cs | 18 ++ .../Assets/AzureBlobAssetStore.cs | 23 +- .../Assets/GoogleCloudAssetStore.cs | 19 +- .../Assets/FolderAssetStore.cs | 18 +- .../Assets/IAssetStore.cs | 11 +- .../Apps/AppLanguagesController.cs | 4 +- .../Controllers/Backup/BackupController.cs | 105 ++++++++ .../Controllers/Backup/Models/BackupJobDto.cs | 38 +++ .../Assets/AssetCommandMiddlewareTests.cs | 13 +- .../AssetUserPictureStoreTests.cs | 9 +- .../Assets/AssetStoreTests.cs | 12 +- 36 files changed, 1243 insertions(+), 61 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Backup/BackupGrain.cs create mode 100644 src/Squidex.Domain.Apps.Backup/EventStreamWriter.cs create mode 100644 src/Squidex.Domain.Apps.Backup/IBackupArchiveLocation.cs create mode 100644 src/Squidex.Domain.Apps.Backup/IBackupGrain.cs create mode 100644 src/Squidex.Domain.Apps.Backup/IBackupJob.cs create mode 100644 src/Squidex.Domain.Apps.Backup/Squidex.Domain.Apps.Backup.csproj create mode 100644 src/Squidex.Domain.Apps.Backup/State/BackupState.cs create mode 100644 src/Squidex.Domain.Apps.Backup/State/BackupStateJob.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IBackupArchiveLocation.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Contents/IContentScheduleItem.cs create mode 100644 src/Squidex.Domain.Apps.Entities/Contents/State/ContentStateScheduleItem.cs create mode 100644 src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs create mode 100644 src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs create mode 100644 src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs create mode 100644 src/Squidex/Areas/Api/Controllers/Backup/Models/BackupJobDto.cs diff --git a/src/Squidex.Domain.Apps.Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Backup/BackupGrain.cs new file mode 100644 index 000000000..7390cf21f --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/BackupGrain.cs @@ -0,0 +1,242 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NodaTime; +using Orleans; +using Orleans.Concurrency; +using Squidex.Domain.Apps.Backup.State; +using Squidex.Domain.Apps.Events.Assets; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Log; +using Squidex.Infrastructure.Orleans; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Backup +{ + [Reentrant] + public sealed class BackupGrain : Grain, IBackupGrain + { + private const int MaxBackups = 10; + private readonly IClock clock; + private readonly IAssetStore assetStore; + private readonly IEventDataFormatter eventDataFormatter; + private readonly ISemanticLog log; + private readonly IEventStore eventStore; + private readonly IBackupArchiveLocation backupArchiveLocation; + private readonly IStore store; + private CancellationTokenSource currentTask; + private BackupStateJob currentJob; + private Guid appId; + private BackupState state; + private IPersistence persistence; + + public BackupGrain( + IAssetStore assetStore, + IBackupArchiveLocation backupArchiveLocation, + IClock clock, + IEventStore eventStore, + IEventDataFormatter eventDataFormatter, + ISemanticLog log, + IStore store) + { + Guard.NotNull(assetStore, nameof(assetStore)); + Guard.NotNull(backupArchiveLocation, nameof(backupArchiveLocation)); + Guard.NotNull(clock, nameof(clock)); + Guard.NotNull(eventStore, nameof(eventStore)); + Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter)); + Guard.NotNull(store, nameof(store)); + Guard.NotNull(log, nameof(log)); + + this.assetStore = assetStore; + this.backupArchiveLocation = backupArchiveLocation; + this.clock = clock; + this.eventStore = eventStore; + this.eventDataFormatter = eventDataFormatter; + this.store = store; + this.log = log; + } + + public override Task OnActivateAsync() + { + return OnActivateAsync(this.GetPrimaryKey()); + } + + public async Task OnActivateAsync(Guid appId) + { + this.appId = appId; + + persistence = store.WithSnapshots(GetType(), appId, s => state = s); + + await ReadAsync(); + await CleanupAsync(); + } + + private async Task ReadAsync() + { + await persistence.ReadAsync(); + } + + private async Task WriteAsync() + { + await persistence.WriteSnapshotAsync(state); + } + + private async Task CleanupAsync() + { + var hasUpdated = false; + + foreach (var job in state.Jobs) + { + if (!job.Stopped.HasValue) + { + await CleanupAsync(job); + + job.Stopped = clock.GetCurrentInstant(); + job.Failed = true; + + hasUpdated = true; + } + } + + if (hasUpdated) + { + await WriteAsync(); + } + } + + private async Task CleanupAsync(BackupStateJob job) + { + await backupArchiveLocation.DeleteArchiveAsync(job.Id); + } + + public async Task StartNewAsync() + { + if (currentTask != null) + { + throw new DomainException("Another backup process is already running."); + } + + if (state.Jobs.Count >= MaxBackups) + { + throw new DomainException($"You cannot have more than {MaxBackups} backups."); + } + + var job = new BackupStateJob { Id = Guid.NewGuid(), Started = clock.GetCurrentInstant() }; + + currentTask = new CancellationTokenSource(); + currentJob = job; + + state.Jobs.Add(job); + + await WriteAsync(); + + try + { + using (var stream = await backupArchiveLocation.OpenStreamAsync(job.Id)) + { + using (var writer = new EventStreamWriter(stream)) + { + await eventStore.QueryAsync(async @event => + { + var eventData = @event.Data; + + if (eventData.Type == nameof(AssetCreated) || + eventData.Type == nameof(AssetUpdated)) + { + var parsedEvent = eventDataFormatter.Parse(eventData); + + var assetVersion = 0L; + var assetId = Guid.Empty; + + if (parsedEvent.Payload is AssetCreated assetCreated) + { + assetId = assetCreated.AssetId; + assetVersion = assetCreated.FileVersion; + } + + if (parsedEvent.Payload is AssetUpdated asetUpdated) + { + assetId = asetUpdated.AssetId; + assetVersion = asetUpdated.FileVersion; + } + + await writer.WriteEventAsync(eventData, async attachmentStream => + { + await assetStore.DownloadAsync(assetId.ToString(), assetVersion, null, attachmentStream); + }); + } + else + { + await writer.WriteEventAsync(eventData); + } + }, "AppId", appId, null, currentTask.Token); + } + + stream.Position = 0; + + currentTask.Token.ThrowIfCancellationRequested(); + + await assetStore.UploadAsync(job.Id.ToString(), 0, null, stream); + + currentTask.Token.ThrowIfCancellationRequested(); + } + } + catch + { + job.Failed = true; + } + finally + { + job.Stopped = clock.GetCurrentInstant(); + + await WriteAsync(); + + currentTask = null; + currentJob = null; + } + } + + public async Task DeleteAsync(Guid id) + { + var job = state.Jobs.FirstOrDefault(x => x.Id == id); + + if (job == null) + { + throw new DomainObjectNotFoundException(id.ToString(), typeof(IBackupJob)); + } + + if (currentJob == job) + { + currentTask?.Cancel(); + } + else + { + state.Jobs.Remove(job); + + await WriteAsync(); + await CleanupAsync(job); + } + } + + public Task>> GetStateAsync() + { + return Task.FromResult(new J>(state.Jobs.OfType().ToList())); + } + + private bool IsRunning() + { + return state.Jobs.Any(x => !x.Stopped.HasValue); + } + } +} diff --git a/src/Squidex.Domain.Apps.Backup/EventStreamWriter.cs b/src/Squidex.Domain.Apps.Backup/EventStreamWriter.cs new file mode 100644 index 000000000..210132ddc --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/EventStreamWriter.cs @@ -0,0 +1,80 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; + +namespace Squidex.Domain.Apps.Backup +{ + public sealed class EventStreamWriter : DisposableObjectBase + { + private const int MaxItemsPerFolder = 1000; + private readonly StreamWriter streamWriter; + private readonly ZipArchive archive; + private int writtenEvents; + private int writtenAttachments; + + public EventStreamWriter(Stream stream) + { + archive = new ZipArchive(stream, ZipArchiveMode.Update, true); + } + + public async Task WriteEventAsync(EventData eventData, Func attachment = null) + { + var eventObject = + new JObject( + new JProperty("type", eventData.Type), + new JProperty("payload", eventData.Payload), + new JProperty("metadata", eventData.Metadata)); + + var eventFolder = writtenEvents / MaxItemsPerFolder; + var eventPath = $"events/{eventFolder}/{writtenEvents}.json"; + var eventEntry = archive.GetEntry(eventPath) ?? archive.CreateEntry(eventPath); + + using (var stream = eventEntry.Open()) + { + using (var textWriter = new StreamWriter(stream)) + { + using (var jsonWriter = new JsonTextWriter(textWriter)) + { + await eventObject.WriteToAsync(jsonWriter); + } + } + } + + writtenEvents++; + + if (attachment != null) + { + var attachmentFolder = writtenAttachments / MaxItemsPerFolder; + var attachmentPath = $"attachments/{attachmentFolder}/{writtenEvents}.blob"; + var attachmentEntry = archive.GetEntry(attachmentPath) ?? archive.CreateEntry(attachmentPath); + + using (var stream = eventEntry.Open()) + { + await attachment(stream); + } + + writtenAttachments++; + } + } + + protected override void DisposeObject(bool disposing) + { + if (disposing) + { + archive.Dispose(); + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Backup/IBackupArchiveLocation.cs b/src/Squidex.Domain.Apps.Backup/IBackupArchiveLocation.cs new file mode 100644 index 000000000..300abbbe9 --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/IBackupArchiveLocation.cs @@ -0,0 +1,13 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Squidex.Domain.Apps.Backup +{ + public interface IBackupArchiveLocation + { + Task OpenStreamAsync(Guid backupId); + + Task DeleteArchiveAsync(Guid backupId); + } +} diff --git a/src/Squidex.Domain.Apps.Backup/IBackupGrain.cs b/src/Squidex.Domain.Apps.Backup/IBackupGrain.cs new file mode 100644 index 000000000..1c982f853 --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/IBackupGrain.cs @@ -0,0 +1,24 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans; +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Backup +{ + public interface IBackupGrain : IGrainWithGuidKey + { + Task StartNewAsync(); + + Task DeleteAsync(Guid id); + + Task>> GetStateAsync(); + } +} diff --git a/src/Squidex.Domain.Apps.Backup/IBackupJob.cs b/src/Squidex.Domain.Apps.Backup/IBackupJob.cs new file mode 100644 index 000000000..53a93f9b8 --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/IBackupJob.cs @@ -0,0 +1,25 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using NodaTime; + +namespace Squidex.Domain.Apps.Backup +{ + public interface IBackupJob + { + Guid Id { get; } + + Instant Started { get; } + + Instant? Stopped { get; } + + bool Failed { get; } + + string DownloadPath { get; } + } +} diff --git a/src/Squidex.Domain.Apps.Backup/Squidex.Domain.Apps.Backup.csproj b/src/Squidex.Domain.Apps.Backup/Squidex.Domain.Apps.Backup.csproj new file mode 100644 index 000000000..cbe70395d --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/Squidex.Domain.Apps.Backup.csproj @@ -0,0 +1,22 @@ + + + netstandard2.0 + + + + + + + + + + + + + + ..\..\Squidex.ruleset + + + + + diff --git a/src/Squidex.Domain.Apps.Backup/State/BackupState.cs b/src/Squidex.Domain.Apps.Backup/State/BackupState.cs new file mode 100644 index 000000000..e6000eea7 --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/State/BackupState.cs @@ -0,0 +1,18 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Squidex.Domain.Apps.Backup.State +{ + public sealed class BackupState + { + [JsonProperty] + public List Jobs { get; set; } = new List(); + } +} diff --git a/src/Squidex.Domain.Apps.Backup/State/BackupStateJob.cs b/src/Squidex.Domain.Apps.Backup/State/BackupStateJob.cs new file mode 100644 index 000000000..410709f13 --- /dev/null +++ b/src/Squidex.Domain.Apps.Backup/State/BackupStateJob.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Newtonsoft.Json; +using NodaTime; + +namespace Squidex.Domain.Apps.Backup.State +{ + public sealed class BackupStateJob : IBackupJob + { + [JsonProperty] + public Guid Id { get; set; } + + [JsonProperty] + public Instant Started { get; set; } + + [JsonProperty] + public Instant? Stopped { get; set; } + + [JsonProperty] + public string DownloadPath { get; set; } + + [JsonProperty] + public bool Failed { get; set; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs b/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs index f1c779aae..2f8d229e4 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs @@ -25,7 +25,7 @@ using Squidex.Shared.Users; namespace Squidex.Domain.Apps.Entities.Apps { - public class AppGrain : DomainObjectGrain, IAppGrain + public class AppGrain : SquidexDomainObjectGrain, IAppGrain { private readonly InitialPatterns initialPatterns; private readonly IAppProvider appProvider; diff --git a/src/Squidex.Domain.Apps.Entities/Assets/AssetCommandMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Assets/AssetCommandMiddleware.cs index a50f1196d..bb349d793 100644 --- a/src/Squidex.Domain.Apps.Entities/Assets/AssetCommandMiddleware.cs +++ b/src/Squidex.Domain.Apps.Entities/Assets/AssetCommandMiddleware.cs @@ -41,18 +41,18 @@ namespace Squidex.Domain.Apps.Entities.Assets { createAsset.ImageInfo = await assetThumbnailGenerator.GetImageInfoAsync(createAsset.File.OpenRead()); - await assetStore.UploadTemporaryAsync(context.ContextId.ToString(), createAsset.File.OpenRead()); + await assetStore.UploadAsync(context.ContextId.ToString(), createAsset.File.OpenRead()); try { var result = await ExecuteCommandAsync(createAsset) as AssetSavedResult; context.Complete(EntityCreatedResult.Create(createAsset.AssetId, result.Version)); - await assetStore.CopyTemporaryAsync(context.ContextId.ToString(), createAsset.AssetId.ToString(), result.FileVersion, null); + await assetStore.CopyAsync(context.ContextId.ToString(), createAsset.AssetId.ToString(), result.FileVersion, null); } finally { - await assetStore.DeleteTemporaryAsync(context.ContextId.ToString()); + await assetStore.DeleteAsync(context.ContextId.ToString()); } break; @@ -62,18 +62,18 @@ namespace Squidex.Domain.Apps.Entities.Assets { updateAsset.ImageInfo = await assetThumbnailGenerator.GetImageInfoAsync(updateAsset.File.OpenRead()); - await assetStore.UploadTemporaryAsync(context.ContextId.ToString(), updateAsset.File.OpenRead()); + await assetStore.UploadAsync(context.ContextId.ToString(), updateAsset.File.OpenRead()); try { var result = await ExecuteCommandAsync(updateAsset) as AssetSavedResult; context.Complete(result); - await assetStore.CopyTemporaryAsync(context.ContextId.ToString(), updateAsset.AssetId.ToString(), result.FileVersion, null); + await assetStore.CopyAsync(context.ContextId.ToString(), updateAsset.AssetId.ToString(), result.FileVersion, null); } finally { - await assetStore.DeleteTemporaryAsync(context.ContextId.ToString()); + await assetStore.DeleteAsync(context.ContextId.ToString()); } break; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs new file mode 100644 index 000000000..2596d4647 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -0,0 +1,242 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NodaTime; +using Orleans; +using Orleans.Concurrency; +using Squidex.Domain.Apps.Entities.Backup.State; +using Squidex.Domain.Apps.Events.Assets; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Log; +using Squidex.Infrastructure.Orleans; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + [Reentrant] + public sealed class BackupGrain : Grain, IBackupGrain + { + private const int MaxBackups = 10; + private readonly IClock clock; + private readonly IAssetStore assetStore; + private readonly IEventDataFormatter eventDataFormatter; + private readonly ISemanticLog log; + private readonly IEventStore eventStore; + private readonly IBackupArchiveLocation backupArchiveLocation; + private readonly IStore store; + private CancellationTokenSource currentTask; + private BackupStateJob currentJob; + private Guid appId; + private BackupState state; + private IPersistence persistence; + + public BackupGrain( + IAssetStore assetStore, + IBackupArchiveLocation backupArchiveLocation, + IClock clock, + IEventStore eventStore, + IEventDataFormatter eventDataFormatter, + ISemanticLog log, + IStore store) + { + Guard.NotNull(assetStore, nameof(assetStore)); + Guard.NotNull(backupArchiveLocation, nameof(backupArchiveLocation)); + Guard.NotNull(clock, nameof(clock)); + Guard.NotNull(eventStore, nameof(eventStore)); + Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter)); + Guard.NotNull(store, nameof(store)); + Guard.NotNull(log, nameof(log)); + + this.assetStore = assetStore; + this.backupArchiveLocation = backupArchiveLocation; + this.clock = clock; + this.eventStore = eventStore; + this.eventDataFormatter = eventDataFormatter; + this.store = store; + this.log = log; + } + + public override Task OnActivateAsync() + { + return OnActivateAsync(this.GetPrimaryKey()); + } + + public async Task OnActivateAsync(Guid appId) + { + this.appId = appId; + + persistence = store.WithSnapshots(GetType(), appId, s => state = s); + + await ReadAsync(); + await CleanupAsync(); + } + + private async Task ReadAsync() + { + await persistence.ReadAsync(); + } + + private async Task WriteAsync() + { + await persistence.WriteSnapshotAsync(state); + } + + private async Task CleanupAsync() + { + var hasUpdated = false; + + foreach (var job in state.Jobs) + { + if (!job.Stopped.HasValue) + { + await CleanupAsync(job); + + job.Stopped = clock.GetCurrentInstant(); + job.Failed = true; + + hasUpdated = true; + } + } + + if (hasUpdated) + { + await WriteAsync(); + } + } + + private async Task CleanupAsync(BackupStateJob job) + { + await backupArchiveLocation.DeleteArchiveAsync(job.Id); + } + + public async Task StartNewAsync() + { + if (currentTask != null) + { + throw new DomainException("Another backup process is already running."); + } + + if (state.Jobs.Count >= MaxBackups) + { + throw new DomainException($"You cannot have more than {MaxBackups} backups."); + } + + var job = new BackupStateJob { Id = Guid.NewGuid(), Started = clock.GetCurrentInstant() }; + + currentTask = new CancellationTokenSource(); + currentJob = job; + + state.Jobs.Add(job); + + await WriteAsync(); + + try + { + using (var stream = await backupArchiveLocation.OpenStreamAsync(job.Id)) + { + using (var writer = new EventStreamWriter(stream)) + { + await eventStore.QueryAsync(async @event => + { + var eventData = @event.Data; + + if (eventData.Type == nameof(AssetCreated) || + eventData.Type == nameof(AssetUpdated)) + { + var parsedEvent = eventDataFormatter.Parse(eventData); + + var assetVersion = 0L; + var assetId = Guid.Empty; + + if (parsedEvent.Payload is AssetCreated assetCreated) + { + assetId = assetCreated.AssetId; + assetVersion = assetCreated.FileVersion; + } + + if (parsedEvent.Payload is AssetUpdated asetUpdated) + { + assetId = asetUpdated.AssetId; + assetVersion = asetUpdated.FileVersion; + } + + await writer.WriteEventAsync(eventData, async attachmentStream => + { + await assetStore.DownloadAsync(assetId.ToString(), assetVersion, null, attachmentStream); + }); + } + else + { + await writer.WriteEventAsync(eventData); + } + }, "AppId", appId, null, currentTask.Token); + } + + stream.Position = 0; + + currentTask.Token.ThrowIfCancellationRequested(); + + await assetStore.UploadAsync(job.Id.ToString(), 0, null, stream, currentTask.Token); + + currentTask.Token.ThrowIfCancellationRequested(); + } + } + catch + { + job.Failed = true; + } + finally + { + job.Stopped = clock.GetCurrentInstant(); + + await WriteAsync(); + + currentTask = null; + currentJob = null; + } + } + + public async Task DeleteAsync(Guid id) + { + var job = state.Jobs.FirstOrDefault(x => x.Id == id); + + if (job == null) + { + throw new DomainObjectNotFoundException(id.ToString(), typeof(IBackupJob)); + } + + if (currentJob == job) + { + currentTask?.Cancel(); + } + else + { + state.Jobs.Remove(job); + + await WriteAsync(); + await CleanupAsync(job); + } + } + + public Task>> GetStateAsync() + { + return Task.FromResult(new J>(state.Jobs.OfType().ToList())); + } + + private bool IsRunning() + { + return state.Jobs.Any(x => !x.Stopped.HasValue); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs new file mode 100644 index 000000000..a6a8e29c2 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs @@ -0,0 +1,79 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public sealed class EventStreamWriter : DisposableObjectBase + { + private const int MaxItemsPerFolder = 1000; + private readonly ZipArchive archive; + private int writtenEvents; + private int writtenAttachments; + + public EventStreamWriter(Stream stream) + { + archive = new ZipArchive(stream, ZipArchiveMode.Update, true); + } + + public async Task WriteEventAsync(EventData eventData, Func attachment = null) + { + var eventObject = + new JObject( + new JProperty("type", eventData.Type), + new JProperty("payload", eventData.Payload), + new JProperty("metadata", eventData.Metadata)); + + var eventFolder = writtenEvents / MaxItemsPerFolder; + var eventPath = $"events/{eventFolder}/{writtenEvents}.json"; + var eventEntry = archive.GetEntry(eventPath) ?? archive.CreateEntry(eventPath); + + using (var stream = eventEntry.Open()) + { + using (var textWriter = new StreamWriter(stream)) + { + using (var jsonWriter = new JsonTextWriter(textWriter)) + { + await eventObject.WriteToAsync(jsonWriter); + } + } + } + + writtenEvents++; + + if (attachment != null) + { + var attachmentFolder = writtenAttachments / MaxItemsPerFolder; + var attachmentPath = $"attachments/{attachmentFolder}/{writtenEvents}.blob"; + var attachmentEntry = archive.GetEntry(attachmentPath) ?? archive.CreateEntry(attachmentPath); + + using (var stream = eventEntry.Open()) + { + await attachment(stream); + } + + writtenAttachments++; + } + } + + protected override void DisposeObject(bool disposing) + { + if (disposing) + { + archive.Dispose(); + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupArchiveLocation.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupArchiveLocation.cs new file mode 100644 index 000000000..298372a9c --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupArchiveLocation.cs @@ -0,0 +1,20 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface IBackupArchiveLocation + { + Task OpenStreamAsync(Guid backupId); + + Task DeleteArchiveAsync(Guid backupId); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs new file mode 100644 index 000000000..3a2736204 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs @@ -0,0 +1,24 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans; +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface IBackupGrain : IGrainWithGuidKey + { + Task StartNewAsync(); + + Task DeleteAsync(Guid id); + + Task>> GetStateAsync(); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs new file mode 100644 index 000000000..472c0e295 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs @@ -0,0 +1,23 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using NodaTime; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public interface IBackupJob + { + Guid Id { get; } + + Instant Started { get; } + + Instant? Stopped { get; } + + bool Failed { get; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs new file mode 100644 index 000000000..5729030d5 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs @@ -0,0 +1,18 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Squidex.Domain.Apps.Entities.Backup.State +{ + public sealed class BackupState + { + [JsonProperty] + public List Jobs { get; set; } = new List(); + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs new file mode 100644 index 000000000..771843464 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs @@ -0,0 +1,28 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Newtonsoft.Json; +using NodaTime; + +namespace Squidex.Domain.Apps.Entities.Backup.State +{ + public sealed class BackupStateJob : IBackupJob + { + [JsonProperty] + public Guid Id { get; set; } + + [JsonProperty] + public Instant Started { get; set; } + + [JsonProperty] + public Instant? Stopped { get; set; } + + [JsonProperty] + public bool Failed { get; set; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs b/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs new file mode 100644 index 000000000..faa099b4b --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs @@ -0,0 +1,44 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Threading.Tasks; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Backup +{ + public sealed class TempFolderBackupArchiveLocation : IBackupArchiveLocation + { + public Task OpenStreamAsync(Guid backupId) + { + var tempFile = GetTempFile(backupId); + + return Task.FromResult(new FileStream(tempFile, FileMode.Open, FileAccess.ReadWrite)); + } + + public Task DeleteArchiveAsync(Guid backupId) + { + var tempFile = GetTempFile(backupId); + + try + { + File.Delete(tempFile); + } + catch (IOException) + { + } + + return TaskHelper.Done; + } + + private static string GetTempFile(Guid backupId) + { + return Path.Combine(Path.GetTempPath(), backupId.ToString()); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/ContentGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/ContentGrain.cs index 97334f087..0c060bfdc 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/ContentGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/ContentGrain.cs @@ -24,7 +24,7 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Contents { - public class ContentGrain : DomainObjectGrain, IContentGrain + public class ContentGrain : SquidexDomainObjectGrain, IContentGrain { private readonly IAppProvider appProvider; private readonly IAssetRepository assetRepository; diff --git a/src/Squidex.Domain.Apps.Entities/Contents/IContentScheduleItem.cs b/src/Squidex.Domain.Apps.Entities/Contents/IContentScheduleItem.cs new file mode 100644 index 000000000..0cd2e4257 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/IContentScheduleItem.cs @@ -0,0 +1,22 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using NodaTime; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Contents +{ + public interface IContentScheduleItem + { + Status ScheduledTo { get; } + + Instant ScheduledAt { get; } + + RefToken ScheduledBy { get; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/State/ContentStateScheduleItem.cs b/src/Squidex.Domain.Apps.Entities/Contents/State/ContentStateScheduleItem.cs new file mode 100644 index 000000000..a73f6f0a4 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/State/ContentStateScheduleItem.cs @@ -0,0 +1,26 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Newtonsoft.Json; +using NodaTime; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Contents.State +{ + public sealed class ContentStateScheduleItem : IContentScheduleItem + { + [JsonProperty] + public Instant ScheduledAt { get; set; } + + [JsonProperty] + public RefToken ScheduledBy { get; set; } + + [JsonProperty] + public Status ScheduledTo { get; set; } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Rules/RuleGrain.cs b/src/Squidex.Domain.Apps.Entities/Rules/RuleGrain.cs index ba316f6a2..7bc1bf8a2 100644 --- a/src/Squidex.Domain.Apps.Entities/Rules/RuleGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Rules/RuleGrain.cs @@ -21,7 +21,7 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Rules { - public sealed class RuleGrain : DomainObjectGrain, IRuleGrain + public sealed class RuleGrain : SquidexDomainObjectGrain, IRuleGrain { private readonly IAppProvider appProvider; diff --git a/src/Squidex.Domain.Apps.Entities/Schemas/SchemaGrain.cs b/src/Squidex.Domain.Apps.Entities/Schemas/SchemaGrain.cs index baf1ef9ec..db9d92be5 100644 --- a/src/Squidex.Domain.Apps.Entities/Schemas/SchemaGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Schemas/SchemaGrain.cs @@ -24,7 +24,7 @@ using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.Schemas { - public class SchemaGrain : DomainObjectGrain, ISchemaGrain + public class SchemaGrain : SquidexDomainObjectGrain, ISchemaGrain { private readonly IAppProvider appProvider; private readonly FieldRegistry registry; diff --git a/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs b/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs new file mode 100644 index 000000000..9808cedb4 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Squidex.Domain.Apps.Events; +using Squidex.Infrastructure.Commands; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities +{ + public abstract class SquidexDomainObjectGrain : DomainObjectGrain where T : IDomainState, new() + { + protected SquidexDomainObjectGrain(IStore store) + : base(store) + { + } + + public override void RaiseEvent(Envelope @event) + { + if (@event.Payload is AppEvent appEvent) + { + @event.SetAppId(appEvent.AppId.Id); + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/SquidexEntities.cs b/src/Squidex.Domain.Apps.Entities/SquidexEntities.cs index c65786a8d..8d82fce42 100644 --- a/src/Squidex.Domain.Apps.Entities/SquidexEntities.cs +++ b/src/Squidex.Domain.Apps.Entities/SquidexEntities.cs @@ -9,7 +9,7 @@ using System.Reflection; namespace Squidex.Domain.Apps.Entities { - public sealed class SquidexEntities + public static class SquidexEntities { public static readonly Assembly Assembly = typeof(SquidexEntities).Assembly; } diff --git a/src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs b/src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs new file mode 100644 index 000000000..315972adf --- /dev/null +++ b/src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs @@ -0,0 +1,18 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Squidex.Infrastructure.EventSourcing; + +namespace Squidex.Domain.Apps.Events.Contents +{ + [EventType(nameof(ContentStatusChanged))] + public sealed class ContentScheduleItemRemoved : ContentEvent + { + public Guid ScheduleItemId { get; set; } + } +} diff --git a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs index 258a37ae2..e19b9af8a 100644 --- a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs +++ b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs @@ -7,6 +7,7 @@ using System; using System.IO; +using System.Threading; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; @@ -56,7 +57,7 @@ namespace Squidex.Infrastructure.Assets return new Uri(blobContainer.StorageUri.PrimaryUri, $"/{containerName}/{blobName}").ToString(); } - public async Task CopyTemporaryAsync(string name, string id, long version, string suffix) + public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { var blobName = GetObjectName(id, version, suffix); var blobRef = blobContainer.GetBlobReference(blobName); @@ -65,12 +66,14 @@ namespace Squidex.Infrastructure.Assets try { - await blobRef.StartCopyAsync(tempBlob.Uri); + await blobRef.StartCopyAsync(tempBlob.Uri, null, null, null, null, ct); while (blobRef.CopyState.Status == CopyStatus.Pending) { + ct.ThrowIfCancellationRequested(); + await Task.Delay(50); - await blobRef.FetchAttributesAsync(); + await blobRef.FetchAttributesAsync(null, null, null, ct); } if (blobRef.CopyState.Status != CopyStatus.Success) @@ -84,14 +87,14 @@ namespace Squidex.Infrastructure.Assets } } - public async Task DownloadAsync(string id, long version, string suffix, Stream stream) + public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var blobName = GetObjectName(id, version, suffix); var blobRef = blobContainer.GetBlockBlobReference(blobName); try { - await blobRef.DownloadToStreamAsync(stream); + await blobRef.DownloadToStreamAsync(stream, null, null, null, ct); } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404) { @@ -99,7 +102,7 @@ namespace Squidex.Infrastructure.Assets } } - public async Task UploadAsync(string id, long version, string suffix, Stream stream) + public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var blobName = GetObjectName(id, version, suffix); var blobRef = blobContainer.GetBlockBlobReference(blobName); @@ -107,18 +110,18 @@ namespace Squidex.Infrastructure.Assets blobRef.Metadata[AssetVersion] = version.ToString(); blobRef.Metadata[AssetId] = id; - await blobRef.UploadFromStreamAsync(stream); + await blobRef.UploadFromStreamAsync(stream, null, null, null, ct); await blobRef.SetMetadataAsync(); } - public async Task UploadTemporaryAsync(string name, Stream stream) + public async Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) { var tempBlob = blobContainer.GetBlockBlobReference(name); - await tempBlob.UploadFromStreamAsync(stream); + await tempBlob.UploadFromStreamAsync(stream, null, null, null, ct); } - public async Task DeleteTemporaryAsync(string name) + public async Task DeleteAsync(string name) { var tempBlob = blobContainer.GetBlockBlobReference(name); diff --git a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs index ec1bad1c7..8df8c6094 100644 --- a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs +++ b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs @@ -9,6 +9,7 @@ using System; using System.IO; using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; using Google; using Google.Cloud.Storage.V1; @@ -48,25 +49,25 @@ namespace Squidex.Infrastructure.Assets return $"https://storage.cloud.google.com/{bucketName}/{objectName}"; } - public Task UploadTemporaryAsync(string name, Stream stream) + public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) { - return storageClient.UploadObjectAsync(bucketName, name, "application/octet-stream", stream); + return storageClient.UploadObjectAsync(bucketName, name, "application/octet-stream", stream, cancellationToken: ct); } - public async Task UploadAsync(string id, long version, string suffix, Stream stream) + public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var objectName = GetObjectName(id, version, suffix); - await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream); + await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, cancellationToken: ct); } - public async Task CopyTemporaryAsync(string name, string id, long version, string suffix) + public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { var objectName = GetObjectName(id, version, suffix); try { - await storageClient.CopyObjectAsync(bucketName, name, bucketName, objectName); + await storageClient.CopyObjectAsync(bucketName, name, bucketName, objectName, cancellationToken: ct); } catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound) { @@ -74,13 +75,13 @@ namespace Squidex.Infrastructure.Assets } } - public async Task DownloadAsync(string id, long version, string suffix, Stream stream) + public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var objectName = GetObjectName(id, version, suffix); try { - await storageClient.DownloadObjectAsync(bucketName, objectName, stream); + await storageClient.DownloadObjectAsync(bucketName, objectName, stream, cancellationToken: ct); } catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound) { @@ -88,7 +89,7 @@ namespace Squidex.Infrastructure.Assets } } - public async Task DeleteTemporaryAsync(string name) + public async Task DeleteAsync(string name) { try { diff --git a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs index 396bffbc7..d908cbf24 100644 --- a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Tasks; @@ -15,6 +16,7 @@ namespace Squidex.Infrastructure.Assets { public sealed class FolderAssetStore : IAssetStore, IInitializable { + private const int BufferSize = 81920; private readonly ISemanticLog log; private readonly DirectoryInfo directory; @@ -57,27 +59,27 @@ namespace Squidex.Infrastructure.Assets return file.FullName; } - public async Task UploadTemporaryAsync(string name, Stream stream) + public async Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) { var file = GetFile(name); using (var fileStream = file.OpenWrite()) { - await stream.CopyToAsync(fileStream); + await stream.CopyToAsync(fileStream, BufferSize, ct); } } - public async Task UploadAsync(string id, long version, string suffix, Stream stream) + public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var file = GetFile(id, version, suffix); using (var fileStream = file.OpenWrite()) { - await stream.CopyToAsync(fileStream); + await stream.CopyToAsync(fileStream, BufferSize, ct); } } - public async Task DownloadAsync(string id, long version, string suffix, Stream stream) + public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)) { var file = GetFile(id, version, suffix); @@ -85,7 +87,7 @@ namespace Squidex.Infrastructure.Assets { using (var fileStream = file.OpenRead()) { - await fileStream.CopyToAsync(stream); + await fileStream.CopyToAsync(stream, BufferSize, ct); } } catch (FileNotFoundException ex) @@ -94,7 +96,7 @@ namespace Squidex.Infrastructure.Assets } } - public Task CopyTemporaryAsync(string name, string id, long version, string suffix) + public Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)) { try { @@ -110,7 +112,7 @@ namespace Squidex.Infrastructure.Assets } } - public Task DeleteTemporaryAsync(string name) + public Task DeleteAsync(string name) { try { diff --git a/src/Squidex.Infrastructure/Assets/IAssetStore.cs b/src/Squidex.Infrastructure/Assets/IAssetStore.cs index 4c2ab9358..9e8cd3bcd 100644 --- a/src/Squidex.Infrastructure/Assets/IAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/IAssetStore.cs @@ -6,6 +6,7 @@ // ========================================================================== using System.IO; +using System.Threading; using System.Threading.Tasks; namespace Squidex.Infrastructure.Assets @@ -14,14 +15,14 @@ namespace Squidex.Infrastructure.Assets { string GenerateSourceUrl(string id, long version, string suffix); - Task CopyTemporaryAsync(string name, string id, long version, string suffix); + Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken)); - Task DownloadAsync(string id, long version, string suffix, Stream stream); + Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)); - Task UploadTemporaryAsync(string name, Stream stream); + Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)); - Task UploadAsync(string id, long version, string suffix, Stream stream); + Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken)); - Task DeleteTemporaryAsync(string name); + Task DeleteAsync(string name); } } \ No newline at end of file diff --git a/src/Squidex/Areas/Api/Controllers/Apps/AppLanguagesController.cs b/src/Squidex/Areas/Api/Controllers/Apps/AppLanguagesController.cs index 53dff0972..cd89c8035 100644 --- a/src/Squidex/Areas/Api/Controllers/Apps/AppLanguagesController.cs +++ b/src/Squidex/Areas/Api/Controllers/Apps/AppLanguagesController.cs @@ -98,7 +98,7 @@ namespace Squidex.Areas.Api.Controllers.Apps /// /// 204 => Language updated. /// 400 => Language object is invalid. - /// 404 => App not found. + /// 404 => Language or app not found. /// [MustBeAppEditor] [HttpPut] @@ -118,7 +118,7 @@ namespace Squidex.Areas.Api.Controllers.Apps /// The language to delete from the app. /// /// 204 => Language deleted. - /// 404 => App not found. + /// 404 => Language or app not found. /// [MustBeAppEditor] [HttpDelete] diff --git a/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs b/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs new file mode 100644 index 000000000..4cb34a669 --- /dev/null +++ b/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs @@ -0,0 +1,105 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using NSwag.Annotations; +using Orleans; +using Squidex.Areas.Api.Controllers.Backup.Models; +using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Infrastructure.Commands; +using Squidex.Infrastructure.Reflection; +using Squidex.Pipeline; + +namespace Squidex.Areas.Api.Controllers.Backup +{ + /// + /// Manages backups for app. + /// + [ApiAuthorize] + [ApiExceptionFilter] + [AppApi] + [MustBeAppOwner] + [SwaggerTag(nameof(Backup))] + public class BackupController : ApiController + { + private readonly IGrainFactory grainFactory; + + public BackupController(ICommandBus commandBus, IGrainFactory grainFactory) + : base(commandBus) + { + this.grainFactory = grainFactory; + } + + /// + /// Get all backup jobs. + /// + /// The name of the app. + /// + /// 200 => Backups returned. + /// 404 => App not found. + /// + [HttpGet] + [Route("apps/{app}/backups/")] + [ProducesResponseType(typeof(List), 200)] + [ApiCosts(0)] + public async Task GetJobs(string app) + { + var backupGrain = grainFactory.GetGrain(App.Id); + + var jobs = await backupGrain.GetStateAsync(); + + return Ok(jobs.Value.Select(x => SimpleMapper.Map(x, new BackupJobDto())).ToList()); + } + + /// + /// Start a new backup. + /// + /// The name of the app. + /// + /// 204 => Backup started. + /// 404 => App not found. + /// + [HttpPost] + [Route("apps/{app}/backups/")] + [ProducesResponseType(typeof(List), 200)] + [ApiCosts(0)] + public async Task PostBackup(string app) + { + var backupGrain = grainFactory.GetGrain(App.Id); + + await backupGrain.StartNewAsync(); + + return NoContent(); + } + + /// + /// Delete a backup. + /// + /// The name of the app. + /// The id of the backup to delete. + /// + /// 204 => Backup started. + /// 404 => Backup or app not found. + /// + [HttpPost] + [Route("apps/{app}/backups/{id}")] + [ProducesResponseType(typeof(List), 200)] + [ApiCosts(0)] + public async Task PostBackup(string app, Guid id) + { + var backupGrain = grainFactory.GetGrain(App.Id); + + await backupGrain.StartNewAsync(); + + return NoContent(); + } + } +} diff --git a/src/Squidex/Areas/Api/Controllers/Backup/Models/BackupJobDto.cs b/src/Squidex/Areas/Api/Controllers/Backup/Models/BackupJobDto.cs new file mode 100644 index 000000000..a7dd12f07 --- /dev/null +++ b/src/Squidex/Areas/Api/Controllers/Backup/Models/BackupJobDto.cs @@ -0,0 +1,38 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.ComponentModel.DataAnnotations; +using NodaTime; + +namespace Squidex.Areas.Api.Controllers.Backup.Models +{ + public sealed class BackupJobDto + { + /// + /// The id of the backup job. + /// + [Required] + public Guid Id { get; set; } + + /// + /// The time when the job has been started. + /// + [Required] + public Instant Started { get; set; } + + /// + /// The time when the job has been stopped. + /// + public Instant? Stopped { get; } + + /// + /// Indicates if the job has failed. + /// + public bool Failed { get; } + } +} diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetCommandMiddlewareTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetCommandMiddlewareTests.cs index 6882a0450..cf87d3020 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetCommandMiddlewareTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetCommandMiddlewareTests.cs @@ -7,6 +7,7 @@ using System; using System.IO; +using System.Threading; using System.Threading.Tasks; using FakeItEasy; using Orleans; @@ -89,21 +90,21 @@ namespace Squidex.Domain.Apps.Entities.Assets private void SetupStore(long version, Guid commitId) { - A.CallTo(() => assetStore.UploadTemporaryAsync(commitId.ToString(), stream)) + A.CallTo(() => assetStore.UploadAsync(commitId.ToString(), stream, CancellationToken.None)) .Returns(TaskHelper.Done); - A.CallTo(() => assetStore.CopyTemporaryAsync(commitId.ToString(), assetId.ToString(), version, null)) + A.CallTo(() => assetStore.CopyAsync(commitId.ToString(), assetId.ToString(), version, null, CancellationToken.None)) .Returns(TaskHelper.Done); - A.CallTo(() => assetStore.DeleteTemporaryAsync(commitId.ToString())) + A.CallTo(() => assetStore.DeleteAsync(commitId.ToString())) .Returns(TaskHelper.Done); } private void AssertAssetHasBeenUploaded(long version, Guid commitId) { - A.CallTo(() => assetStore.UploadTemporaryAsync(commitId.ToString(), stream)) + A.CallTo(() => assetStore.UploadAsync(commitId.ToString(), stream, CancellationToken.None)) .MustHaveHappened(); - A.CallTo(() => assetStore.CopyTemporaryAsync(commitId.ToString(), assetId.ToString(), version, null)) + A.CallTo(() => assetStore.CopyAsync(commitId.ToString(), assetId.ToString(), version, null, CancellationToken.None)) .MustHaveHappened(); - A.CallTo(() => assetStore.DeleteTemporaryAsync(commitId.ToString())) + A.CallTo(() => assetStore.DeleteAsync(commitId.ToString())) .MustHaveHappened(); } diff --git a/tests/Squidex.Domain.Users.Tests/AssetUserPictureStoreTests.cs b/tests/Squidex.Domain.Users.Tests/AssetUserPictureStoreTests.cs index 396a3efde..c5d169050 100644 --- a/tests/Squidex.Domain.Users.Tests/AssetUserPictureStoreTests.cs +++ b/tests/Squidex.Domain.Users.Tests/AssetUserPictureStoreTests.cs @@ -7,6 +7,7 @@ using System; using System.IO; +using System.Threading; using System.Threading.Tasks; using FakeItEasy; using Squidex.Infrastructure.Assets; @@ -33,18 +34,18 @@ namespace Squidex.Domain.Users { var stream = new MemoryStream(); - A.CallTo(() => assetStore.UploadAsync(userId, 0, "picture", stream)) + A.CallTo(() => assetStore.UploadAsync(userId, 0, "picture", stream, CancellationToken.None)) .Returns(TaskHelper.Done); await sut.UploadAsync(userId, stream); - A.CallTo(() => assetStore.UploadAsync(userId, 0, "picture", stream)).MustHaveHappened(); + A.CallTo(() => assetStore.UploadAsync(userId, 0, "picture", stream, CancellationToken.None)).MustHaveHappened(); } [Fact] public async Task Should_invoke_asset_store_to_download_picture() { - A.CallTo(() => assetStore.DownloadAsync(userId, 0, "picture", A.Ignored)) + A.CallTo(() => assetStore.DownloadAsync(userId, 0, "picture", A.Ignored, CancellationToken.None)) .Invokes(async (string id, long version, string suffix, Stream stream) => { await stream.WriteAsync(new byte[] { 1, 2, 3, 4 }, 0, 4); @@ -55,7 +56,7 @@ namespace Squidex.Domain.Users Assert.Equal(0, result.Position); Assert.Equal(4, result.Length); - A.CallTo(() => assetStore.DownloadAsync(userId, 0, "picture", A.Ignored)).MustHaveHappened(); + A.CallTo(() => assetStore.DownloadAsync(userId, 0, "picture", A.Ignored, CancellationToken.None)).MustHaveHappened(); } } } diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index 64316f5b5..657331efc 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -43,7 +43,7 @@ namespace Squidex.Infrastructure.Assets { ((IInitializable)Sut).Initialize(); - return Assert.ThrowsAsync(() => Sut.CopyTemporaryAsync(Id(), Id(), 1, null)); + return Assert.ThrowsAsync(() => Sut.CopyAsync(Id(), Id(), 1, null)); } [Fact] @@ -73,8 +73,8 @@ namespace Squidex.Infrastructure.Assets var assetId = Id(); var assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); - await Sut.UploadTemporaryAsync(tempId, assetData); - await Sut.CopyTemporaryAsync(tempId, assetId, 1, "suffix"); + await Sut.UploadAsync(tempId, assetData); + await Sut.CopyAsync(tempId, assetId, 1, "suffix"); var readData = new MemoryStream(); @@ -92,9 +92,9 @@ namespace Squidex.Infrastructure.Assets var assetData = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 }); - await Sut.UploadTemporaryAsync(tempId, assetData); - await Sut.DeleteTemporaryAsync(tempId); - await Sut.DeleteTemporaryAsync(tempId); + await Sut.UploadAsync(tempId, assetData); + await Sut.DeleteAsync(tempId); + await Sut.DeleteAsync(tempId); } private static string Id() From ab4bc05cd99f88453eaae2064ecc02fdb06e092f Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 Mar 2018 17:45:11 +0100 Subject: [PATCH 2/7] Download Endpoint and UI page --- .../Backup/BackupGrain.cs | 7 +- .../Backup/IBackupGrain.cs | 2 +- .../Backup/State/BackupState.cs | 2 +- .../Backup/TempFolderBackupArchiveLocation.cs | 2 +- .../SquidexDomainObjectGrain.cs | 2 + .../Contents/ContentScheduleItemRemoved.cs | 18 ----- .../States/MongoSnapshotStore.cs | 2 +- src/Squidex.Infrastructure/States/Store.cs | 4 +- .../States/StoreExtensions.cs | 12 +-- .../Assets/AssetContentController.cs | 16 ++-- .../Controllers/Backup/BackupController.cs | 34 +++++++-- src/Squidex/Config/Domain/EntitiesServices.cs | 4 + src/Squidex/Config/Domain/StoreServices.cs | 5 ++ .../app/features/settings/declarations.ts | 1 + src/Squidex/app/features/settings/module.ts | 6 ++ .../pages/backups/backups-page.component.html | 34 +++++++++ .../pages/backups/backups-page.component.scss | 2 + .../pages/backups/backups-page.component.ts | 49 ++++++++++++ .../settings/settings-area.component.html | 8 +- src/Squidex/app/shared/declarations-base.ts | 1 + src/Squidex/app/shared/module.ts | 2 + .../app/shared/services/backups.service.ts | 76 +++++++++++++++++++ 22 files changed, 241 insertions(+), 48 deletions(-) delete mode 100644 src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs create mode 100644 src/Squidex/app/features/settings/pages/backups/backups-page.component.html create mode 100644 src/Squidex/app/features/settings/pages/backups/backups-page.component.scss create mode 100644 src/Squidex/app/features/settings/pages/backups/backups-page.component.ts create mode 100644 src/Squidex/app/shared/services/backups.service.ts diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index 2596d4647..3ea5d0ad3 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -14,6 +14,7 @@ using NodaTime; using Orleans; using Orleans.Concurrency; using Squidex.Domain.Apps.Entities.Backup.State; +using Squidex.Domain.Apps.Events; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; @@ -38,7 +39,7 @@ namespace Squidex.Domain.Apps.Entities.Backup private CancellationTokenSource currentTask; private BackupStateJob currentJob; private Guid appId; - private BackupState state; + private BackupState state = new BackupState(); private IPersistence persistence; public BackupGrain( @@ -120,7 +121,7 @@ namespace Squidex.Domain.Apps.Entities.Backup await backupArchiveLocation.DeleteArchiveAsync(job.Id); } - public async Task StartNewAsync() + public async Task RunAsync() { if (currentTask != null) { @@ -180,7 +181,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { await writer.WriteEventAsync(eventData); } - }, "AppId", appId, null, currentTask.Token); + }, SquidexHeaders.AppId, appId.ToString(), null, currentTask.Token); } stream.Position = 0; diff --git a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs index 3a2736204..21f66e2ff 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/IBackupGrain.cs @@ -15,7 +15,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { public interface IBackupGrain : IGrainWithGuidKey { - Task StartNewAsync(); + Task RunAsync(); Task DeleteAsync(Guid id); diff --git a/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs index 5729030d5..e75eef133 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/State/BackupState.cs @@ -13,6 +13,6 @@ namespace Squidex.Domain.Apps.Entities.Backup.State public sealed class BackupState { [JsonProperty] - public List Jobs { get; set; } = new List(); + public List Jobs { get; } = new List(); } } diff --git a/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs b/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs index faa099b4b..b710a46ac 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs @@ -18,7 +18,7 @@ namespace Squidex.Domain.Apps.Entities.Backup { var tempFile = GetTempFile(backupId); - return Task.FromResult(new FileStream(tempFile, FileMode.Open, FileAccess.ReadWrite)); + return Task.FromResult(new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite)); } public Task DeleteArchiveAsync(Guid backupId) diff --git a/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs b/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs index 9808cedb4..b3fd6f7d8 100644 --- a/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/SquidexDomainObjectGrain.cs @@ -26,6 +26,8 @@ namespace Squidex.Domain.Apps.Entities { @event.SetAppId(appEvent.AppId.Id); } + + base.RaiseEvent(@event); } } } diff --git a/src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs b/src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs deleted file mode 100644 index 315972adf..000000000 --- a/src/Squidex.Domain.Apps.Events/Contents/ContentScheduleItemRemoved.cs +++ /dev/null @@ -1,18 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using Squidex.Infrastructure.EventSourcing; - -namespace Squidex.Domain.Apps.Events.Contents -{ - [EventType(nameof(ContentStatusChanged))] - public sealed class ContentScheduleItemRemoved : ContentEvent - { - public Guid ScheduleItemId { get; set; } - } -} diff --git a/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs b/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs index c90506746..19f641f38 100644 --- a/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs @@ -32,7 +32,7 @@ namespace Squidex.Infrastructure.States public async Task<(T Value, long Version)> ReadAsync(TKey key) { var existing = - await Collection.Find(x => Equals(x.Id, key)) + await Collection.Find(x => x.Id.Equals(key)) .FirstOrDefaultAsync(); if (existing != null) diff --git a/src/Squidex.Infrastructure/States/Store.cs b/src/Squidex.Infrastructure/States/Store.cs index a3d6b1bc6..3a010d3ed 100644 --- a/src/Squidex.Infrastructure/States/Store.cs +++ b/src/Squidex.Infrastructure/States/Store.cs @@ -32,12 +32,12 @@ namespace Squidex.Infrastructure.States public IPersistence WithSnapshots(Type owner, TKey key, Func applySnapshot) { - return CreatePersistence(owner, key, PersistenceMode.Snapshots, applySnapshot, null); + return CreatePersistence(owner, key, PersistenceMode.Snapshots, applySnapshot, null); } public IPersistence WithSnapshotsAndEventSourcing(Type owner, TKey key, Func applySnapshot, Func, Task> applyEvent) { - return CreatePersistence(owner, key, PersistenceMode.SnapshotsAndEventSourcing, applySnapshot, applyEvent); + return CreatePersistence(owner, key, PersistenceMode.SnapshotsAndEventSourcing, applySnapshot, applyEvent); } public IPersistence WithEventSourcing(Type owner, TKey key, Func, Task> applyEvent) diff --git a/src/Squidex.Infrastructure/States/StoreExtensions.cs b/src/Squidex.Infrastructure/States/StoreExtensions.cs index 3cee24593..1164807ad 100644 --- a/src/Squidex.Infrastructure/States/StoreExtensions.cs +++ b/src/Squidex.Infrastructure/States/StoreExtensions.cs @@ -21,12 +21,12 @@ namespace Squidex.Infrastructure.States public static IPersistence WithSnapshots(this IStore store, TKey key, Func applySnapshot) { - return store.WithSnapshots(typeof(TOwner), key, applySnapshot); + return store.WithSnapshots(typeof(TOwner), key, applySnapshot); } public static IPersistence WithSnapshotsAndEventSourcing(this IStore store, TKey key, Func applySnapshot, Func, Task> applyEvent) { - return store.WithSnapshotsAndEventSourcing(typeof(TOwner), key, applySnapshot, applyEvent); + return store.WithSnapshotsAndEventSourcing(typeof(TOwner), key, applySnapshot, applyEvent); } public static IPersistence WithEventSourcing(this IStore store, Type owner, TKey key, Action> applyEvent) @@ -36,12 +36,12 @@ namespace Squidex.Infrastructure.States public static IPersistence WithSnapshots(this IStore store, Type owner, TKey key, Action applySnapshot) { - return store.WithSnapshots(owner, key, applySnapshot.ToAsync()); + return store.WithSnapshots(owner, key, applySnapshot.ToAsync()); } public static IPersistence WithSnapshotsAndEventSourcing(this IStore store, Type owner, TKey key, Action applySnapshot, Action> applyEvent) { - return store.WithSnapshotsAndEventSourcing(owner, key, applySnapshot.ToAsync(), applyEvent.ToAsync()); + return store.WithSnapshotsAndEventSourcing(owner, key, applySnapshot.ToAsync(), applyEvent.ToAsync()); } public static IPersistence WithEventSourcing(this IStore store, TKey key, Action> applyEvent) @@ -51,12 +51,12 @@ namespace Squidex.Infrastructure.States public static IPersistence WithSnapshots(this IStore store, TKey key, Action applySnapshot) { - return store.WithSnapshots(typeof(TOwner), key, applySnapshot.ToAsync()); + return store.WithSnapshots(typeof(TOwner), key, applySnapshot.ToAsync()); } public static IPersistence WithSnapshotsAndEventSourcing(this IStore store, TKey key, Action applySnapshot, Action> applyEvent) { - return store.WithSnapshotsAndEventSourcing(typeof(TOwner), key, applySnapshot.ToAsync(), applyEvent.ToAsync()); + return store.WithSnapshotsAndEventSourcing(typeof(TOwner), key, applySnapshot.ToAsync(), applyEvent.ToAsync()); } } } diff --git a/src/Squidex/Areas/Api/Controllers/Assets/AssetContentController.cs b/src/Squidex/Areas/Api/Controllers/Assets/AssetContentController.cs index f0e4c4d03..9e1ec9fdd 100644 --- a/src/Squidex/Areas/Api/Controllers/Assets/AssetContentController.cs +++ b/src/Squidex/Areas/Api/Controllers/Assets/AssetContentController.cs @@ -27,18 +27,18 @@ namespace Squidex.Areas.Api.Controllers.Assets [SwaggerTag(nameof(Assets))] public sealed class AssetContentController : ApiController { - private readonly IAssetStore assetStorage; + private readonly IAssetStore assetStore; private readonly IAssetRepository assetRepository; private readonly IAssetThumbnailGenerator assetThumbnailGenerator; public AssetContentController( ICommandBus commandBus, - IAssetStore assetStorage, + IAssetStore assetStore, IAssetRepository assetRepository, IAssetThumbnailGenerator assetThumbnailGenerator) : base(commandBus) { - this.assetStorage = assetStorage; + this.assetStore = assetStore; this.assetRepository = assetRepository; this.assetThumbnailGenerator = assetThumbnailGenerator; } @@ -53,7 +53,7 @@ namespace Squidex.Areas.Api.Controllers.Assets /// The target height of the asset, if it is an image. /// The resize mode when the width and height is defined. /// - /// 200 => Asset found and content or (resize) image returned. + /// 200 => Asset found and content or (resized) image returned. /// 404 => Asset or app not found. /// [HttpGet] @@ -79,7 +79,7 @@ namespace Squidex.Areas.Api.Controllers.Assets try { - await assetStorage.DownloadAsync(assetId, entity.FileVersion, assetSuffix, bodyStream); + await assetStore.DownloadAsync(assetId, entity.FileVersion, assetSuffix, bodyStream); } catch (AssetNotFoundException) { @@ -87,13 +87,13 @@ namespace Squidex.Areas.Api.Controllers.Assets { using (var destinationStream = GetTempStream()) { - await assetStorage.DownloadAsync(assetId, entity.FileVersion, null, sourceStream); + await assetStore.DownloadAsync(assetId, entity.FileVersion, null, sourceStream); sourceStream.Position = 0; await assetThumbnailGenerator.CreateThumbnailAsync(sourceStream, destinationStream, width, height, mode); destinationStream.Position = 0; - await assetStorage.UploadAsync(assetId, entity.FileVersion, assetSuffix, destinationStream); + await assetStore.UploadAsync(assetId, entity.FileVersion, assetSuffix, destinationStream); destinationStream.Position = 0; await destinationStream.CopyToAsync(bodyStream); @@ -103,7 +103,7 @@ namespace Squidex.Areas.Api.Controllers.Assets } else { - await assetStorage.DownloadAsync(assetId, entity.FileVersion, null, bodyStream); + await assetStore.DownloadAsync(assetId, entity.FileVersion, null, bodyStream); } }); } diff --git a/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs b/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs index 4cb34a669..f4991ceec 100644 --- a/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs +++ b/src/Squidex/Areas/Api/Controllers/Backup/BackupController.cs @@ -14,8 +14,10 @@ using NSwag.Annotations; using Orleans; using Squidex.Areas.Api.Controllers.Backup.Models; using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Infrastructure.Assets; using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Reflection; +using Squidex.Infrastructure.Tasks; using Squidex.Pipeline; namespace Squidex.Areas.Api.Controllers.Backup @@ -31,11 +33,13 @@ namespace Squidex.Areas.Api.Controllers.Backup public class BackupController : ApiController { private readonly IGrainFactory grainFactory; + private readonly IAssetStore assetStore; - public BackupController(ICommandBus commandBus, IGrainFactory grainFactory) + public BackupController(ICommandBus commandBus, IGrainFactory grainFactory, IAssetStore assetStore) : base(commandBus) { this.grainFactory = grainFactory; + this.assetStore = assetStore; } /// @@ -71,15 +75,33 @@ namespace Squidex.Areas.Api.Controllers.Backup [Route("apps/{app}/backups/")] [ProducesResponseType(typeof(List), 200)] [ApiCosts(0)] - public async Task PostBackup(string app) + public IActionResult PostBackup(string app) { var backupGrain = grainFactory.GetGrain(App.Id); - await backupGrain.StartNewAsync(); + backupGrain.RunAsync().Forget(); return NoContent(); } + /// + /// Get the backup content. + /// + /// The name of the app. + /// The id of the asset. + /// + /// 200 => Backup found and content returned. + /// 404 => Backup or app not found. + /// + [HttpGet] + [Route("apps/{app}/backups/{id}")] + [ProducesResponseType(200)] + [ApiCosts(0.5)] + public IActionResult GetBackupContent(string app, Guid id) + { + return new FileCallbackResult("application/zip", "Backup.zip", bodyStream => assetStore.DownloadAsync(id.ToString(), 0, null, bodyStream)); + } + /// /// Delete a backup. /// @@ -89,15 +111,15 @@ namespace Squidex.Areas.Api.Controllers.Backup /// 204 => Backup started. /// 404 => Backup or app not found. /// - [HttpPost] + [HttpDelete] [Route("apps/{app}/backups/{id}")] [ProducesResponseType(typeof(List), 200)] [ApiCosts(0)] - public async Task PostBackup(string app, Guid id) + public async Task DeleteBackup(string app, Guid id) { var backupGrain = grainFactory.GetGrain(App.Id); - await backupGrain.StartNewAsync(); + await backupGrain.DeleteAsync(id); return NoContent(); } diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index cabc20e5c..158c3afdf 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -19,6 +19,7 @@ using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Apps.Commands; using Squidex.Domain.Apps.Entities.Apps.Templates; using Squidex.Domain.Apps.Entities.Assets; +using Squidex.Domain.Apps.Entities.Backup; using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Contents.Commands; using Squidex.Domain.Apps.Entities.Contents.Edm; @@ -50,6 +51,9 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); + services.AddSingletonAs() + .As(); + services.AddSingletonAs() .As(); diff --git a/src/Squidex/Config/Domain/StoreServices.cs b/src/Squidex/Config/Domain/StoreServices.cs index 537b06688..f4f8e56ba 100644 --- a/src/Squidex/Config/Domain/StoreServices.cs +++ b/src/Squidex/Config/Domain/StoreServices.cs @@ -18,6 +18,7 @@ using Squidex.Domain.Apps.Entities.Apps.Repositories; using Squidex.Domain.Apps.Entities.Apps.State; using Squidex.Domain.Apps.Entities.Assets.Repositories; using Squidex.Domain.Apps.Entities.Assets.State; +using Squidex.Domain.Apps.Entities.Backup.State; using Squidex.Domain.Apps.Entities.Contents.Repositories; using Squidex.Domain.Apps.Entities.Contents.State; using Squidex.Domain.Apps.Entities.History; @@ -69,6 +70,10 @@ namespace Squidex.Config.Domain .As() .As(); + services.AddSingletonAs(c => new MongoSnapshotStore(mongoDatabase, c.GetRequiredService())) + .As>() + .As(); + services.AddSingletonAs(c => new MongoSnapshotStore(mongoDatabase, c.GetRequiredService())) .As>() .As(); diff --git a/src/Squidex/app/features/settings/declarations.ts b/src/Squidex/app/features/settings/declarations.ts index f9932b668..66917745d 100644 --- a/src/Squidex/app/features/settings/declarations.ts +++ b/src/Squidex/app/features/settings/declarations.ts @@ -5,6 +5,7 @@ * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. */ +export * from './pages/backups/backups-page.component'; export * from './pages/clients/client.component'; export * from './pages/clients/clients-page.component'; export * from './pages/contributors/contributors-page.component'; diff --git a/src/Squidex/app/features/settings/module.ts b/src/Squidex/app/features/settings/module.ts index 519f86b39..40b5ccd3c 100644 --- a/src/Squidex/app/features/settings/module.ts +++ b/src/Squidex/app/features/settings/module.ts @@ -17,6 +17,7 @@ import { } from 'shared'; import { + BackupsPageComponent, ClientComponent, ClientsPageComponent, ContributorsPageComponent, @@ -45,6 +46,10 @@ const routes: Routes = [ path: 'more', component: MorePageComponent }, + { + path: 'backups', + component: BackupsPageComponent + }, { path: 'clients', component: ClientsPageComponent, @@ -137,6 +142,7 @@ const routes: Routes = [ RouterModule.forChild(routes) ], declarations: [ + BackupsPageComponent, ClientComponent, ClientsPageComponent, ContributorsPageComponent, diff --git a/src/Squidex/app/features/settings/pages/backups/backups-page.component.html b/src/Squidex/app/features/settings/pages/backups/backups-page.component.html new file mode 100644 index 000000000..fdeb85a57 --- /dev/null +++ b/src/Squidex/app/features/settings/pages/backups/backups-page.component.html @@ -0,0 +1,34 @@ + + + +
+
+
+ +
+ +

Backups

+
+ + + + +
+ +
+
+ + + + + + + + + +
{{backup.started | sqxFullDateTime }}
+
+
+
\ No newline at end of file diff --git a/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss b/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss new file mode 100644 index 000000000..fbb752506 --- /dev/null +++ b/src/Squidex/app/features/settings/pages/backups/backups-page.component.scss @@ -0,0 +1,2 @@ +@import '_vars'; +@import '_mixins'; \ No newline at end of file diff --git a/src/Squidex/app/features/settings/pages/backups/backups-page.component.ts b/src/Squidex/app/features/settings/pages/backups/backups-page.component.ts new file mode 100644 index 000000000..851cacb5f --- /dev/null +++ b/src/Squidex/app/features/settings/pages/backups/backups-page.component.ts @@ -0,0 +1,49 @@ +/* + * Squidex Headless CMS + * + * @license + * Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved. + */ + +import { Component } from '@angular/core'; +import { Observable } from 'rxjs'; + +import { AppContext, BackupsService } from 'shared'; + +@Component({ + selector: 'sqx-backups-page', + styleUrls: ['./backups-page.component.scss'], + templateUrl: './backups-page.component.html', + providers: [ + AppContext + ] +}) +export class BackupsPageComponent { + public backups = + Observable.timer(0, 5000) + .switchMap(t => this.backupsService.getBackups(this.ctx.appName)); + + constructor(public readonly ctx: AppContext, + private readonly backupsService: BackupsService + ) { + } + + public startBackup() { + this.backupsService.postBackup(this.ctx.appName) + .subscribe(() => { + this.ctx.notifyInfo('Backup started.'); + }, error => { + this.ctx.notifyError(error); + }); + } + + public deleteBackup(id: string) { + this.backupsService.deleteBackup(this.ctx.appName, id) + .subscribe(() => { + this.ctx.notifyInfo('Backup deleting.'); + }, error => { + this.ctx.notifyError(error); + }); + } +} + diff --git a/src/Squidex/app/features/settings/settings-area.component.html b/src/Squidex/app/features/settings/settings-area.component.html index 0f5a96935..7b3900ea5 100644 --- a/src/Squidex/app/features/settings/settings-area.component.html +++ b/src/Squidex/app/features/settings/settings-area.component.html @@ -14,6 +14,12 @@