Browse Source

Merge branch 'master' of github.com:Squidex/squidex

pull/312/head
Sebastian Stehle 7 years ago
parent
commit
27655a3e02
  1. 4
      src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs
  2. 5
      src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs
  3. 15
      src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs
  4. 5
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs
  5. 9
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs
  6. 5
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentPublishedCollection.cs
  7. 15
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs
  8. 5
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs
  9. 5
      src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs
  10. 5
      src/Squidex.Domain.Apps.Entities.MongoDb/Rules/MongoRuleEventRepository.cs
  11. 3
      src/Squidex.Domain.Apps.Entities/AppProvider.cs
  12. 8
      src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs
  13. 190
      src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs
  14. 9
      src/Squidex.Domain.Apps.Entities/Apps/Guards/GuardApp.cs
  15. 38
      src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexCommandMiddleware.cs
  16. 41
      src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs
  17. 6
      src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByNameIndex.cs
  18. 2
      src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByUserIndex.cs
  19. 132
      src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs
  20. 2
      src/Squidex.Domain.Apps.Entities/Assets/Repositories/IAssetRepository.cs
  21. 166
      src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs
  22. 55
      src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs
  23. 60
      src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs
  24. 149
      src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs
  25. 31
      src/Squidex.Domain.Apps.Entities/Backup/BackupRestoreException.cs
  26. 105
      src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs
  27. 79
      src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs
  28. 170
      src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs
  29. 50
      src/Squidex.Domain.Apps.Entities/Backup/Helpers/ArchiveHelper.cs
  30. 66
      src/Squidex.Domain.Apps.Entities/Backup/Helpers/Downloader.cs
  31. 62
      src/Squidex.Domain.Apps.Entities/Backup/Helpers/Safe.cs
  32. 2
      src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs
  33. 21
      src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs
  34. 26
      src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs
  35. 17
      src/Squidex.Domain.Apps.Entities/Backup/JobStatus.cs
  36. 335
      src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs
  37. 2
      src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs
  38. 17
      src/Squidex.Domain.Apps.Entities/Backup/State/RestoreState.cs
  39. 44
      src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs
  40. 4
      src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs
  41. 54
      src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs
  42. 2
      src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs
  43. 2
      src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs
  44. 65
      src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs
  45. 4
      src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs
  46. 7
      src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs
  47. 2
      src/Squidex.Domain.Apps.Entities/Rules/Repositories/IRuleEventRepository.cs
  48. 65
      src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs
  49. 4
      src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs
  50. 7
      src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs
  51. 42
      src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs
  52. 6
      src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs
  53. 14
      src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs
  54. 16
      src/Squidex.Domain.Apps.Entities/Tags/Tag.cs
  55. 30
      src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs
  56. 15
      src/Squidex.Domain.Apps.Entities/Tags/TagSet.cs
  57. 73
      src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs
  58. 1
      src/Squidex.Infrastructure.GetEventStore/EventSourcing/Formatter.cs
  59. 10
      src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs
  60. 66
      src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs
  61. 40
      src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs
  62. 18
      src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs
  63. 10
      src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs
  64. 8
      src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs
  65. 38
      src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs
  66. 17
      src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs
  67. 78
      src/Squidex.Infrastructure/Assets/FolderAssetStore.cs
  68. 6
      src/Squidex.Infrastructure/Assets/IAssetStore.cs
  69. 12
      src/Squidex.Infrastructure/CollectionExtensions.cs
  70. 4
      src/Squidex.Infrastructure/EventSourcing/IEventStore.cs
  71. 7
      src/Squidex.Infrastructure/EventSourcing/StoredEvent.cs
  72. 5
      src/Squidex.Infrastructure/Log/Profiler.cs
  73. 16
      src/Squidex.Infrastructure/RefTokenType.cs
  74. 23
      src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs
  75. 2
      src/Squidex.Infrastructure/States/IPersistence{TState}.cs
  76. 2
      src/Squidex.Infrastructure/States/ISnapshotStore.cs
  77. 2
      src/Squidex.Infrastructure/States/IStore.cs
  78. 2
      src/Squidex.Infrastructure/States/IStreamNameResolver.cs
  79. 13
      src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs
  80. 5
      src/Squidex.Infrastructure/States/Store.cs
  81. 17
      src/Squidex.Infrastructure/States/StoreExtensions.cs
  82. 16
      src/Squidex.Infrastructure/Tasks/TaskExtensions.cs
  83. 4
      src/Squidex/Areas/Api/Controllers/Backups/Models/BackupJobDto.cs
  84. 51
      src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs
  85. 28
      src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs
  86. 71
      src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs
  87. 75
      src/Squidex/Config/Domain/EntitiesServices.cs
  88. 4
      src/Squidex/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddleware.cs
  89. 5
      src/Squidex/app/features/administration/administration-area.component.html
  90. 1
      src/Squidex/app/features/administration/declarations.ts
  91. 6
      src/Squidex/app/features/administration/module.ts
  92. 68
      src/Squidex/app/features/administration/pages/restore/restore-page.component.html
  93. 68
      src/Squidex/app/features/administration/pages/restore/restore-page.component.scss
  94. 68
      src/Squidex/app/features/administration/pages/restore/restore-page.component.ts
  95. 4
      src/Squidex/app/features/administration/services/users.service.ts
  96. 1
      src/Squidex/app/features/rules/pages/events/rule-events-page.component.scss
  97. 8
      src/Squidex/app/features/settings/pages/backups/backups-page.component.html
  98. 8
      src/Squidex/app/features/settings/pages/backups/backups-page.component.scss
  99. 2
      src/Squidex/app/features/settings/pages/languages/language.component.html
  100. 4
      src/Squidex/app/shared/components/history-list.component.scss

4
src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs

@ -209,7 +209,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules
{
if (@event.Actor != null)
{
if (@event.Actor.Type.Equals("client", StringComparison.OrdinalIgnoreCase))
if (@event.Actor.Type.Equals(RefTokenType.Client, StringComparison.OrdinalIgnoreCase))
{
return @event.Actor.ToString();
}
@ -227,7 +227,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules
{
if (@event.Actor != null)
{
if (@event.Actor.Type.Equals("client", StringComparison.OrdinalIgnoreCase))
if (@event.Actor.Type.Equals(RefTokenType.Client, StringComparison.OrdinalIgnoreCase))
{
return @event.Actor.ToString();
}

5
src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs

@ -120,5 +120,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
return assetEntity;
}
}
public Task RemoveAsync(Guid appId)
{
return Collection.DeleteManyAsync(x => x.IndexedAppId == appId);
}
}
}

15
src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs

@ -18,11 +18,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
{
public sealed partial class MongoAssetRepository : ISnapshotStore<AssetState, Guid>
{
Task ISnapshotStore<AssetState, Guid>.ReadAllAsync(Func<AssetState, long, Task> callback)
{
throw new NotSupportedException();
}
public async Task<(AssetState Value, long Version)> ReadAsync(Guid key)
{
using (Profiler.TraceMethod<MongoAssetRepository>())
@ -52,5 +47,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
await Collection.ReplaceOneAsync(x => x.Id == key && x.Version == oldVersion, entity, Upsert);
}
}
Task ISnapshotStore<AssetState, Guid>.ReadAllAsync(Func<AssetState, long, Task> callback)
{
throw new NotSupportedException();
}
Task ISnapshotStore<AssetState, Guid>.RemoveAsync(Guid key)
{
throw new NotSupportedException();
}
}
}

5
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs

@ -117,5 +117,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
Filter.AnyNe(x => x.ReferencedIdsDeleted, id)),
Update.AddToSet(x => x.ReferencedIdsDeleted, id));
}
public Task RemoveAsync(Guid id)
{
return Collection.DeleteOneAsync(x => x.Id == id);
}
}
}

9
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentDraftCollection.cs

@ -58,6 +58,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
return ids.Except(contentEntities.Select(x => Guid.Parse(x["_id"].AsString))).ToList();
}
public async Task<IReadOnlyList<Guid>> QueryIdsAsync(Guid appId)
{
var contentEntities =
await Collection.Find(x => x.IndexedAppId == appId).Only(x => x.Id)
.ToListAsync();
return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList();
}
public Task QueryScheduledWithoutDataAsync(Instant now, Func<IContentEntity, Task> callback)
{
return Collection.Find(x => x.ScheduledAt < now && x.IsDeleted != true)

5
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentPublishedCollection.cs

@ -56,10 +56,5 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
return Collection.ReplaceOneAsync(x => x.Id == content.Id, content, new UpdateOptions { IsUpsert = true });
}
public Task RemoveAsync(Guid id)
{
return Collection.DeleteOneAsync(x => x.Id == id);
}
}
}

15
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs

@ -99,6 +99,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
}
}
public async Task<IReadOnlyList<Guid>> QueryIdsAsync(Guid appId)
{
using (Profiler.TraceMethod<MongoContentRepository>())
{
return await contentsDraft.QueryIdsAsync(appId);
}
}
public async Task QueryScheduledWithoutDataAsync(Instant now, Func<IContentEntity, Task> callback)
{
using (Profiler.TraceMethod<MongoContentRepository>())
@ -107,6 +115,13 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
}
}
public Task RemoveAsync(Guid appId)
{
return Task.WhenAll(
contentsDraft.RemoveAsync(appId),
contentsPublished.RemoveAsync(appId));
}
public Task ClearAsync()
{
return Task.WhenAll(

5
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs

@ -83,6 +83,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
return schema;
}
Task ISnapshotStore<ContentState, Guid>.RemoveAsync(Guid key)
{
throw new NotSupportedException();
}
Task ISnapshotStore<ContentState, Guid>.ReadAllAsync(Func<ContentState, long, Task> callback)
{
throw new NotSupportedException();

5
src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs

@ -112,5 +112,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.History
}
}
}
public Task RemoveAsync(Guid appId)
{
return Collection.DeleteManyAsync(x => x.AppId == appId);
}
}
}

5
src/Squidex.Domain.Apps.Entities.MongoDb/Rules/MongoRuleEventRepository.cs

@ -65,6 +65,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Rules
return ruleEvent;
}
public Task RemoveAsync(Guid appId)
{
return Collection.DeleteManyAsync(x => x.AppId == appId);
}
public async Task<int> CountByAppAsync(Guid appId)
{
return (int)await Collection.CountDocumentsAsync(x => x.AppId == appId);

3
src/Squidex.Domain.Apps.Entities/AppProvider.cs

@ -11,8 +11,11 @@ using System.Linq;
using System.Threading.Tasks;
using Orleans;
using Squidex.Domain.Apps.Entities.Apps;
using Squidex.Domain.Apps.Entities.Apps.Indexes;
using Squidex.Domain.Apps.Entities.Rules;
using Squidex.Domain.Apps.Entities.Rules.Indexes;
using Squidex.Domain.Apps.Entities.Schemas;
using Squidex.Domain.Apps.Entities.Schemas.Indexes;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Caching;
using Squidex.Infrastructure.Log;

8
src/Squidex.Domain.Apps.Entities/Apps/AppGrain.cs

@ -29,7 +29,6 @@ namespace Squidex.Domain.Apps.Entities.Apps
public sealed class AppGrain : SquidexDomainObjectGrain<AppState>, IAppGrain
{
private readonly InitialPatterns initialPatterns;
private readonly IAppProvider appProvider;
private readonly IAppPlansProvider appPlansProvider;
private readonly IAppPlanBillingManager appPlansBillingManager;
private readonly IUserResolver userResolver;
@ -38,20 +37,17 @@ namespace Squidex.Domain.Apps.Entities.Apps
InitialPatterns initialPatterns,
IStore<Guid> store,
ISemanticLog log,
IAppProvider appProvider,
IAppPlansProvider appPlansProvider,
IAppPlanBillingManager appPlansBillingManager,
IUserResolver userResolver)
: base(store, log)
{
Guard.NotNull(initialPatterns, nameof(initialPatterns));
Guard.NotNull(appProvider, nameof(appProvider));
Guard.NotNull(userResolver, nameof(userResolver));
Guard.NotNull(appPlansProvider, nameof(appPlansProvider));
Guard.NotNull(appPlansBillingManager, nameof(appPlansBillingManager));
this.userResolver = userResolver;
this.appProvider = appProvider;
this.appPlansProvider = appPlansProvider;
this.appPlansBillingManager = appPlansBillingManager;
this.initialPatterns = initialPatterns;
@ -64,9 +60,9 @@ namespace Squidex.Domain.Apps.Entities.Apps
switch (command)
{
case CreateApp createApp:
return CreateAsync(createApp, async c =>
return CreateAsync(createApp, c =>
{
await GuardApp.CanCreate(c, appProvider);
GuardApp.CanCreate(c);
Create(c);
});

190
src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs

@ -0,0 +1,190 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using Orleans;
using Squidex.Domain.Apps.Entities.Apps.Indexes;
using Squidex.Domain.Apps.Entities.Apps.State;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Domain.Apps.Events;
using Squidex.Domain.Apps.Events.Apps;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Orleans;
using Squidex.Infrastructure.States;
using Squidex.Shared.Users;
namespace Squidex.Domain.Apps.Entities.Apps
{
public sealed class BackupApps : BackupHandlerWithStore
{
private const string UsersFile = "Users.json";
private readonly IGrainFactory grainFactory;
private readonly IUserResolver userResolver;
private readonly HashSet<string> activeUsers = new HashSet<string>();
private Dictionary<string, string> usersWithEmail = new Dictionary<string, string>();
private Dictionary<string, RefToken> userMapping = new Dictionary<string, RefToken>();
private bool isReserved;
private bool isActorAssigned;
private AppCreated appCreated;
public override string Name { get; } = "Apps";
public BackupApps(IStore<Guid> store, IGrainFactory grainFactory, IUserResolver userResolver)
: base(store)
{
Guard.NotNull(grainFactory, nameof(grainFactory));
Guard.NotNull(userResolver, nameof(userResolver));
this.grainFactory = grainFactory;
this.userResolver = userResolver;
}
public override async Task BackupEventAsync(Envelope<IEvent> @event, Guid appId, BackupWriter writer)
{
if (@event.Payload is AppContributorAssigned appContributorAssigned)
{
var userId = appContributorAssigned.ContributorId;
if (!usersWithEmail.ContainsKey(userId))
{
var user = await userResolver.FindByIdOrEmailAsync(userId);
if (user != null)
{
usersWithEmail.Add(userId, user.Email);
}
}
}
}
public override Task BackupAsync(Guid appId, BackupWriter writer)
{
return WriterUsersAsync(writer);
}
public async override Task RestoreEventAsync(Envelope<IEvent> @event, Guid appId, BackupReader reader, RefToken actor)
{
switch (@event.Payload)
{
case AppCreated appCreated:
{
this.appCreated = appCreated;
await ResolveUsersAsync(reader, actor);
await ReserveAppAsync();
break;
}
case AppContributorAssigned contributorAssigned:
{
if (isActorAssigned)
{
contributorAssigned.ContributorId = MapUser(contributorAssigned.ContributorId, actor).Identifier;
}
else
{
isActorAssigned = true;
contributorAssigned.ContributorId = actor.Identifier;
}
activeUsers.Add(contributorAssigned.ContributorId);
break;
}
case AppContributorRemoved contributorRemoved:
{
contributorRemoved.ContributorId = MapUser(contributorRemoved.ContributorId, actor).Identifier;
activeUsers.Remove(contributorRemoved.ContributorId);
break;
}
}
if (@event.Payload is SquidexEvent squidexEvent)
{
squidexEvent.Actor = MapUser(squidexEvent.Actor.Identifier, actor);
}
}
private async Task ReserveAppAsync()
{
var index = grainFactory.GetGrain<IAppsByNameIndex>(SingleGrain.Id);
if (!(isReserved = await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name)))
{
throw new BackupRestoreException("The app id or name is not available.");
}
}
private RefToken MapUser(string userId, RefToken fallback)
{
return userMapping.GetOrAdd(userId, fallback);
}
private async Task ResolveUsersAsync(BackupReader reader, RefToken actor)
{
await ReadUsersAsync(reader);
foreach (var kvp in usersWithEmail)
{
var user = await userResolver.FindByIdOrEmailAsync(kvp.Value);
if (user != null)
{
userMapping[kvp.Key] = new RefToken(RefTokenType.Subject, user.Id);
}
else
{
userMapping[kvp.Key] = actor;
}
}
}
private async Task ReadUsersAsync(BackupReader reader)
{
var json = await reader.ReadJsonAttachmentAsync(UsersFile);
usersWithEmail = json.ToObject<Dictionary<string, string>>();
}
private Task WriterUsersAsync(BackupWriter writer)
{
var json = JObject.FromObject(usersWithEmail);
return writer.WriteJsonAsync(UsersFile, json);
}
public override async Task CompleteRestoreAsync(Guid appId, BackupReader reader)
{
await RebuildAsync<AppState, AppGrain>(appId, (e, s) => s.Apply(e));
await grainFactory.GetGrain<IAppsByNameIndex>(SingleGrain.Id).AddAppAsync(appCreated.AppId.Id, appCreated.AppId.Name);
foreach (var user in activeUsers)
{
await grainFactory.GetGrain<IAppsByUserIndex>(user).AddAppAsync(appCreated.AppId.Id);
}
}
public override async Task CleanupRestoreAsync(Guid appId)
{
if (isReserved)
{
var index = grainFactory.GetGrain<IAppsByNameIndex>(SingleGrain.Id);
await index.ReserveAppAsync(appCreated.AppId.Id, appCreated.AppId.Name);
}
}
}
}

9
src/Squidex.Domain.Apps.Entities/Apps/Guards/GuardApp.cs

@ -6,7 +6,6 @@
// ==========================================================================
using System;
using System.Threading.Tasks;
using Squidex.Domain.Apps.Core.Apps;
using Squidex.Domain.Apps.Entities.Apps.Commands;
using Squidex.Domain.Apps.Entities.Apps.Services;
@ -16,20 +15,16 @@ namespace Squidex.Domain.Apps.Entities.Apps.Guards
{
public static class GuardApp
{
public static Task CanCreate(CreateApp command, IAppProvider appProvider)
public static void CanCreate(CreateApp command)
{
Guard.NotNull(command, nameof(command));
return Validate.It(() => "Cannot create app.", async e =>
Validate.It(() => "Cannot create app.", e =>
{
if (!command.Name.IsSlug())
{
e("Name must be a valid slug.", nameof(command.Name));
}
else if (await appProvider.GetAppAsync(command.Name) != null)
{
e("An app with the same name already exists.", nameof(command.Name));
}
});
}

38
src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexCommandMiddleware.cs

@ -28,20 +28,44 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes
public async Task HandleAsync(CommandContext context, Func<Task> next)
{
var createApp = context.Command as CreateApp;
var isReserved = false;
try
{
if (createApp != null)
{
isReserved = await index.ReserveAppAsync(createApp.AppId, createApp.Name);
if (!isReserved)
{
var error = new ValidationError("An app with the same name already exists.", nameof(createApp.Name));
throw new ValidationException("Cannot create app.", error);
}
}
await next();
if (context.IsCompleted)
{
switch (context.Command)
if (createApp != null)
{
case CreateApp createApp:
await index.AddAppAsync(createApp.AppId, createApp.Name);
break;
case ArchiveApp archiveApp:
}
else if (context.Command is ArchiveApp archiveApp)
{
await index.RemoveAppAsync(archiveApp.AppId);
break;
}
}
await next();
}
finally
{
if (isReserved && createApp != null)
{
await index.RemoveReservationAsync(createApp.AppId, createApp.Name);
}
}
}
}
}

41
src/Squidex.Domain.Apps.Entities/Apps/Indexes/AppsByNameIndexGrain.cs

@ -12,12 +12,15 @@ using System.Threading.Tasks;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Orleans;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Apps.Indexes
{
public sealed class AppsByNameIndexGrain : GrainOfString, IAppsByNameIndex
{
private readonly IStore<string> store;
private readonly HashSet<Guid> reservedIds = new HashSet<Guid>();
private readonly HashSet<string> reservedNames = new HashSet<string>();
private IPersistence<State> persistence;
private State state = new State();
@ -51,16 +54,52 @@ namespace Squidex.Domain.Apps.Entities.Apps.Indexes
return persistence.WriteSnapshotAsync(state);
}
public Task<bool> ReserveAppAsync(Guid appId, string name)
{
var canReserve =
!state.Apps.ContainsKey(name) &&
!state.Apps.Any(x => x.Value == appId) &&
!reservedIds.Contains(appId) &&
!reservedNames.Contains(name);
if (canReserve)
{
reservedIds.Add(appId);
reservedNames.Add(name);
}
return Task.FromResult(canReserve);
}
public Task RemoveReservationAsync(Guid appId, string name)
{
reservedIds.Remove(appId);
reservedNames.Remove(name);
return TaskHelper.Done;
}
public Task AddAppAsync(Guid appId, string name)
{
state.Apps[name] = appId;
reservedIds.Remove(appId);
reservedNames.Remove(name);
return persistence.WriteSnapshotAsync(state);
}
public Task RemoveAppAsync(Guid appId)
{
state.Apps.Remove(state.Apps.FirstOrDefault(x => x.Value == appId).Key ?? string.Empty);
var name = state.Apps.FirstOrDefault(x => x.Value == appId).Key;
if (!string.IsNullOrWhiteSpace(name))
{
state.Apps.Remove(name);
reservedIds.Remove(appId);
reservedNames.Remove(name);
}
return persistence.WriteSnapshotAsync(state);
}

6
src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByNameIndex.cs

@ -10,16 +10,20 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Orleans;
namespace Squidex.Domain.Apps.Entities.Apps
namespace Squidex.Domain.Apps.Entities.Apps.Indexes
{
public interface IAppsByNameIndex : IGrainWithStringKey
{
Task<bool> ReserveAppAsync(Guid appId, string name);
Task AddAppAsync(Guid appId, string name);
Task RemoveAppAsync(Guid appId);
Task RebuildAsync(Dictionary<string, Guid> apps);
Task RemoveReservationAsync(Guid appId, string name);
Task<Guid> GetAppIdAsync(string name);
Task<List<Guid>> GetAppIdsAsync();

2
src/Squidex.Domain.Apps.Entities/Apps/Indexes/IAppsByUserIndex.cs

@ -10,7 +10,7 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Orleans;
namespace Squidex.Domain.Apps.Entities.Apps
namespace Squidex.Domain.Apps.Entities.Apps.Indexes
{
public interface IAppsByUserIndex : IGrainWithStringKey
{

132
src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs

@ -0,0 +1,132 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using Squidex.Domain.Apps.Entities.Assets.Repositories;
using Squidex.Domain.Apps.Entities.Assets.State;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Domain.Apps.Entities.Tags;
using Squidex.Domain.Apps.Events.Assets;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Assets;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Assets
{
public sealed class BackupAssets : BackupHandlerWithStore
{
private const string TagsFile = "AssetTags.json";
private readonly HashSet<Guid> assetIds = new HashSet<Guid>();
private readonly IAssetStore assetStore;
private readonly IAssetRepository assetRepository;
private readonly ITagService tagService;
public override string Name { get; } = "Assets";
public BackupAssets(IStore<Guid> store,
IAssetStore assetStore,
IAssetRepository assetRepository,
ITagService tagService)
: base(store)
{
Guard.NotNull(assetStore, nameof(assetStore));
Guard.NotNull(assetRepository, nameof(assetRepository));
Guard.NotNull(tagService, nameof(tagService));
this.assetStore = assetStore;
this.assetRepository = assetRepository;
this.tagService = tagService;
}
public override Task BackupAsync(Guid appId, BackupWriter writer)
{
return BackupTagsAsync(appId, writer);
}
public override Task BackupEventAsync(Envelope<IEvent> @event, Guid appId, BackupWriter writer)
{
switch (@event.Payload)
{
case AssetCreated assetCreated:
return WriteAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, writer);
case AssetUpdated assetUpdated:
return WriteAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, writer);
}
return TaskHelper.Done;
}
public override Task RestoreEventAsync(Envelope<IEvent> @event, Guid appId, BackupReader reader, RefToken actor)
{
switch (@event.Payload)
{
case AssetCreated assetCreated:
return ReadAssetAsync(assetCreated.AssetId, assetCreated.FileVersion, reader);
case AssetUpdated assetUpdated:
return ReadAssetAsync(assetUpdated.AssetId, assetUpdated.FileVersion, reader);
}
return TaskHelper.Done;
}
public override async Task RestoreAsync(Guid appId, BackupReader reader)
{
await RestoreTagsAsync(appId, reader);
await RebuildManyAsync(assetIds, id => RebuildAsync<AssetState, AssetGrain>(id, (e, s) => s.Apply(e)));
}
private async Task RestoreTagsAsync(Guid appId, BackupReader reader)
{
var tags = await reader.ReadJsonAttachmentAsync(TagsFile);
await tagService.RebuildTagsAsync(appId, TagGroups.Assets, tags.ToObject<TagSet>());
}
private async Task BackupTagsAsync(Guid appId, BackupWriter writer)
{
var tags = await tagService.GetExportableTagsAsync(appId, TagGroups.Assets);
await writer.WriteJsonAsync(TagsFile, JObject.FromObject(tags));
}
private Task WriteAssetAsync(Guid assetId, long fileVersion, BackupWriter writer)
{
return writer.WriteBlobAsync(GetName(assetId, fileVersion), stream =>
{
return assetStore.DownloadAsync(assetId.ToString(), fileVersion, null, stream);
});
}
private Task ReadAssetAsync(Guid assetId, long fileVersion, BackupReader reader)
{
assetIds.Add(assetId);
return reader.ReadBlobAsync(GetName(reader.OldGuid(assetId), fileVersion), async stream =>
{
try
{
await assetStore.UploadAsync(assetId.ToString(), fileVersion, null, stream);
}
catch (AssetAlreadyExistsException)
{
return;
}
});
}
private static string GetName(Guid assetId, long fileVersion)
{
return $"{assetId}_{fileVersion}.asset";
}
}
}

2
src/Squidex.Domain.Apps.Entities/Assets/Repositories/IAssetRepository.cs

@ -19,5 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Repositories
Task<IResultList<IAssetEntity>> QueryAsync(Guid appId, HashSet<Guid> ids);
Task<IAssetEntity> FindAssetAsync(Guid id);
Task RemoveAsync(Guid appId);
}
}

166
src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs

@ -11,36 +11,37 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using NodaTime;
using Orleans;
using Orleans.Concurrency;
using Squidex.Domain.Apps.Entities.Backup.Helpers;
using Squidex.Domain.Apps.Entities.Backup.State;
using Squidex.Domain.Apps.Events;
using Squidex.Domain.Apps.Events.Assets;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Assets;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Log;
using Squidex.Infrastructure.Orleans;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Backup
{
[Reentrant]
public sealed class BackupGrain : Grain, IBackupGrain
public sealed class BackupGrain : GrainOfGuid, IBackupGrain
{
private const int MaxBackups = 10;
private static readonly Duration UpdateDuration = Duration.FromSeconds(1);
private readonly IClock clock;
private readonly IAssetStore assetStore;
private readonly IBackupArchiveLocation backupArchiveLocation;
private readonly IClock clock;
private readonly IEnumerable<BackupHandler> handlers;
private readonly IEventDataFormatter eventDataFormatter;
private readonly ISemanticLog log;
private readonly IEventStore eventStore;
private readonly IBackupArchiveLocation backupArchiveLocation;
private readonly ISemanticLog log;
private readonly IStore<Guid> store;
private CancellationTokenSource currentTask;
private BackupStateJob currentJob;
private Guid appId;
private BackupState state = new BackupState();
private Guid appId;
private IPersistence<BackupState> persistence;
public BackupGrain(
@ -49,6 +50,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
IClock clock,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter,
IEnumerable<BackupHandler> handlers,
ISemanticLog log,
IStore<Guid> store)
{
@ -57,6 +59,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
Guard.NotNull(clock, nameof(clock));
Guard.NotNull(eventStore, nameof(eventStore));
Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter));
Guard.NotNull(handlers, nameof(handlers));
Guard.NotNull(store, nameof(store));
Guard.NotNull(log, nameof(log));
@ -65,36 +68,28 @@ namespace Squidex.Domain.Apps.Entities.Backup
this.clock = clock;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
this.handlers = handlers;
this.store = store;
this.log = log;
}
public override Task OnActivateAsync()
{
return OnActivateAsync(this.GetPrimaryKey());
}
public async Task OnActivateAsync(Guid appId)
public override async Task OnActivateAsync(Guid key)
{
this.appId = appId;
appId = key;
persistence = store.WithSnapshots<BackupState, Guid>(GetType(), appId, s => state = s);
persistence = store.WithSnapshots<BackupState, Guid>(GetType(), key, s => state = s);
await ReadAsync();
await CleanupAsync();
}
private async Task ReadAsync()
{
await persistence.ReadAsync();
RecoverAfterRestart();
}
private async Task WriteAsync()
private void RecoverAfterRestart()
{
await persistence.WriteSnapshotAsync(state);
RecoverAfterRestartAsync().Forget();
}
private async Task CleanupAsync()
private async Task RecoverAfterRestartAsync()
{
foreach (var job in state.Jobs)
{
@ -102,46 +97,16 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
job.Stopped = clock.GetCurrentInstant();
await CleanupArchiveAsync(job);
await CleanupBackupAsync(job);
await Safe.DeleteAsync(backupArchiveLocation, job.Id, log);
await Safe.DeleteAsync(assetStore, job.Id, log);
job.IsFailed = true;
job.Status = JobStatus.Failed;
await WriteAsync();
}
}
}
private async Task CleanupBackupAsync(BackupStateJob job)
{
try
{
await assetStore.DeleteAsync(job.Id.ToString(), 0, null);
}
catch (Exception ex)
{
log.LogError(ex, w => w
.WriteProperty("action", "deleteBackup")
.WriteProperty("status", "failed")
.WriteProperty("backupId", job.Id.ToString()));
}
}
private async Task CleanupArchiveAsync(BackupStateJob job)
{
try
{
await backupArchiveLocation.DeleteArchiveAsync(job.Id);
}
catch (Exception ex)
{
log.LogError(ex, w => w
.WriteProperty("action", "deleteArchive")
.WriteProperty("status", "failed")
.WriteProperty("backupId", job.Id.ToString()));
}
}
public async Task RunAsync()
{
if (currentTask != null)
@ -154,7 +119,12 @@ namespace Squidex.Domain.Apps.Entities.Backup
throw new DomainException($"You cannot have more than {MaxBackups} backups.");
}
var job = new BackupStateJob { Id = Guid.NewGuid(), Started = clock.GetCurrentInstant() };
var job = new BackupStateJob
{
Id = Guid.NewGuid(),
Started = clock.GetCurrentInstant(),
Status = JobStatus.Started
};
currentTask = new CancellationTokenSource();
currentJob = job;
@ -169,55 +139,34 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
using (var stream = await backupArchiveLocation.OpenStreamAsync(job.Id))
{
using (var writer = new EventStreamWriter(stream))
using (var writer = new BackupWriter(stream, true))
{
await eventStore.QueryAsync(async @event =>
{
var eventData = @event.Data;
if (eventData.Type == "AssetCreatedEvent" ||
eventData.Type == "AssetUpdatedEvent")
await eventStore.QueryAsync(async storedEvent =>
{
var parsedEvent = eventDataFormatter.Parse(eventData);
var @event = eventDataFormatter.Parse(storedEvent.Data);
var assetVersion = 0L;
var assetId = Guid.Empty;
writer.WriteEvent(storedEvent);
if (parsedEvent.Payload is AssetCreated assetCreated)
foreach (var handler in handlers)
{
assetId = assetCreated.AssetId;
assetVersion = assetCreated.FileVersion;
await handler.BackupEventAsync(@event, appId, writer);
}
if (parsedEvent.Payload is AssetUpdated asetUpdated)
{
assetId = asetUpdated.AssetId;
assetVersion = asetUpdated.FileVersion;
}
job.HandledEvents = writer.WrittenEvents;
job.HandledAssets = writer.WrittenAttachments;
await writer.WriteEventAsync(eventData, async attachmentStream =>
{
await assetStore.DownloadAsync(assetId.ToString(), assetVersion, null, attachmentStream);
});
lastTimestamp = await WritePeriodically(lastTimestamp);
}, SquidexHeaders.AppId, appId.ToString(), null, currentTask.Token);
job.HandledAssets++;
}
else
foreach (var handler in handlers)
{
await writer.WriteEventAsync(eventData);
await handler.BackupAsync(appId, writer);
}
job.HandledEvents++;
var now = clock.GetCurrentInstant();
if ((now - lastTimestamp) >= UpdateDuration)
foreach (var handler in handlers)
{
lastTimestamp = now;
await WriteAsync();
await handler.CompleteBackupAsync(appId, writer);
}
}, SquidexHeaders.AppId, appId.ToString(), null, currentTask.Token);
}
stream.Position = 0;
@ -226,6 +175,8 @@ namespace Squidex.Domain.Apps.Entities.Backup
await assetStore.UploadAsync(job.Id.ToString(), 0, null, stream, currentTask.Token);
}
job.Status = JobStatus.Completed;
}
catch (Exception ex)
{
@ -234,11 +185,11 @@ namespace Squidex.Domain.Apps.Entities.Backup
.WriteProperty("status", "failed")
.WriteProperty("backupId", job.Id.ToString()));
job.IsFailed = true;
job.Status = JobStatus.Failed;
}
finally
{
await CleanupArchiveAsync(job);
await Safe.DeleteAsync(backupArchiveLocation, job.Id, log);
job.Stopped = clock.GetCurrentInstant();
@ -249,6 +200,20 @@ namespace Squidex.Domain.Apps.Entities.Backup
}
}
private async Task<Instant> WritePeriodically(Instant lastTimestamp)
{
var now = clock.GetCurrentInstant();
if ((now - lastTimestamp) >= UpdateDuration)
{
lastTimestamp = now;
await WriteAsync();
}
return lastTimestamp;
}
public async Task DeleteAsync(Guid id)
{
var job = state.Jobs.FirstOrDefault(x => x.Id == id);
@ -264,8 +229,8 @@ namespace Squidex.Domain.Apps.Entities.Backup
}
else
{
await CleanupArchiveAsync(job);
await CleanupBackupAsync(job);
await Safe.DeleteAsync(backupArchiveLocation, job.Id, log);
await Safe.DeleteAsync(assetStore, job.Id, log);
state.Jobs.Remove(job);
@ -278,9 +243,14 @@ namespace Squidex.Domain.Apps.Entities.Backup
return J.AsTask(state.Jobs.OfType<IBackupJob>().ToList());
}
private bool IsRunning()
private async Task ReadAsync()
{
return state.Jobs.Any(x => !x.Stopped.HasValue);
await persistence.ReadAsync();
}
private async Task WriteAsync()
{
await persistence.WriteSnapshotAsync(state);
}
}
}

55
src/Squidex.Domain.Apps.Entities/Backup/BackupHandler.cs

@ -0,0 +1,55 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Threading.Tasks;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Backup
{
public abstract class BackupHandler
{
public abstract string Name { get; }
public virtual Task RestoreEventAsync(Envelope<IEvent> @event, Guid appId, BackupReader reader, RefToken actor)
{
return TaskHelper.Done;
}
public virtual Task BackupEventAsync(Envelope<IEvent> @event, Guid appId, BackupWriter writer)
{
return TaskHelper.Done;
}
public virtual Task RestoreAsync(Guid appId, BackupReader reader)
{
return TaskHelper.Done;
}
public virtual Task BackupAsync(Guid appId, BackupWriter writer)
{
return TaskHelper.Done;
}
public virtual Task CleanupRestoreAsync(Guid appId)
{
return TaskHelper.Done;
}
public virtual Task CompleteRestoreAsync(Guid appId, BackupReader reader)
{
return TaskHelper.Done;
}
public virtual Task CompleteBackupAsync(Guid appId, BackupWriter writer)
{
return TaskHelper.Done;
}
}
}

60
src/Squidex.Domain.Apps.Entities/Backup/BackupHandlerWithStore.cs

@ -0,0 +1,60 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
namespace Squidex.Domain.Apps.Entities.Backup
{
public abstract class BackupHandlerWithStore : BackupHandler
{
private readonly IStore<Guid> store;
protected BackupHandlerWithStore(IStore<Guid> store)
{
Guard.NotNull(store, nameof(store));
this.store = store;
}
protected Task RemoveSnapshotAsync<TState>(Guid id)
{
return store.RemoveSnapshotAsync<Guid, TState>(id);
}
protected async Task RebuildManyAsync(IEnumerable<Guid> ids, Func<Guid, Task> action)
{
foreach (var id in ids)
{
await action(id);
}
}
protected async Task RebuildAsync<TState, TGrain>(Guid key, Func<Envelope<IEvent>, TState, TState> func) where TState : IDomainState, new()
{
var state = new TState
{
Version = EtagVersion.Empty
};
var persistence = store.WithSnapshotsAndEventSourcing<TState, Guid>(typeof(TGrain), key, s => state = s, e =>
{
state = func(e, state);
state.Version++;
});
await persistence.ReadAsync();
await persistence.WriteSnapshotAsync(state);
}
}
}

149
src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs

@ -0,0 +1,149 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Squidex.Domain.Apps.Entities.Backup.Archive;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class BackupReader : DisposableObjectBase
{
private static readonly JsonSerializer Serializer = new JsonSerializer();
private readonly GuidMapper guidMapper = new GuidMapper();
private readonly ZipArchive archive;
private int readEvents;
private int readAttachments;
public int ReadEvents
{
get { return readEvents; }
}
public int ReadAttachments
{
get { return readAttachments; }
}
public BackupReader(Stream stream)
{
archive = new ZipArchive(stream, ZipArchiveMode.Read, false);
}
protected override void DisposeObject(bool disposing)
{
if (disposing)
{
archive.Dispose();
}
}
public Guid OldGuid(Guid newId)
{
return guidMapper.OldGuid(newId);
}
public async Task<JToken> ReadJsonAttachmentAsync(string name)
{
Guard.NotNullOrEmpty(name, nameof(name));
var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name));
if (attachmentEntry == null)
{
throw new FileNotFoundException("Cannot find attachment.", name);
}
JToken result;
using (var stream = attachmentEntry.Open())
{
using (var textReader = new StreamReader(stream))
{
using (var jsonReader = new JsonTextReader(textReader))
{
result = await JToken.ReadFromAsync(jsonReader);
guidMapper.NewGuids(result);
}
}
}
readAttachments++;
return result;
}
public async Task ReadBlobAsync(string name, Func<Stream, Task> handler)
{
Guard.NotNullOrEmpty(name, nameof(name));
Guard.NotNull(handler, nameof(handler));
var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name));
if (attachmentEntry == null)
{
throw new FileNotFoundException("Cannot find attachment.", name);
}
using (var stream = attachmentEntry.Open())
{
await handler(stream);
}
readAttachments++;
}
public async Task ReadEventsAsync(IStreamNameResolver streamNameResolver, Func<StoredEvent, Task> handler)
{
Guard.NotNull(handler, nameof(handler));
Guard.NotNull(streamNameResolver, nameof(streamNameResolver));
while (true)
{
var eventEntry = archive.GetEntry(ArchiveHelper.GetEventPath(readEvents));
if (eventEntry == null)
{
break;
}
using (var stream = eventEntry.Open())
{
using (var textReader = new StreamReader(stream))
{
using (var jsonReader = new JsonTextReader(textReader))
{
var storedEvent = Serializer.Deserialize<StoredEvent>(jsonReader);
storedEvent.Data.Payload = guidMapper.NewGuids(storedEvent.Data.Payload);
storedEvent.Data.Metadata = guidMapper.NewGuids(storedEvent.Data.Metadata);
var streamName = streamNameResolver.WithNewId(storedEvent.StreamName, guidMapper.NewGuidString);
storedEvent = new StoredEvent(streamName,
storedEvent.EventPosition,
storedEvent.EventStreamNumber,
storedEvent.Data);
await handler(storedEvent);
}
}
}
readEvents++;
}
}
}
}

31
src/Squidex.Domain.Apps.Entities/Backup/BackupRestoreException.cs

@ -0,0 +1,31 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Runtime.Serialization;
namespace Squidex.Domain.Apps.Entities.Backup
{
[Serializable]
public class BackupRestoreException : Exception
{
public BackupRestoreException(string message)
: base(message)
{
}
public BackupRestoreException(string message, Exception inner)
: base(message, inner)
{
}
protected BackupRestoreException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
}
}

105
src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs

@ -0,0 +1,105 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Squidex.Domain.Apps.Entities.Backup.Archive;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class BackupWriter : DisposableObjectBase
{
private static readonly JsonSerializer Serializer = new JsonSerializer();
private readonly ZipArchive archive;
private int writtenEvents;
private int writtenAttachments;
public int WrittenEvents
{
get { return writtenEvents; }
}
public int WrittenAttachments
{
get { return writtenAttachments; }
}
public BackupWriter(Stream stream, bool keepOpen = false)
{
archive = new ZipArchive(stream, ZipArchiveMode.Create, keepOpen);
}
protected override void DisposeObject(bool disposing)
{
if (disposing)
{
archive.Dispose();
}
}
public async Task WriteJsonAsync(string name, JToken value)
{
Guard.NotNullOrEmpty(name, nameof(name));
var attachmentEntry = archive.CreateEntry(ArchiveHelper.GetAttachmentPath(name));
using (var stream = attachmentEntry.Open())
{
using (var textWriter = new StreamWriter(stream))
{
using (var jsonWriter = new JsonTextWriter(textWriter))
{
await value.WriteToAsync(jsonWriter);
}
}
}
writtenAttachments++;
}
public async Task WriteBlobAsync(string name, Func<Stream, Task> handler)
{
Guard.NotNullOrEmpty(name, nameof(name));
Guard.NotNull(handler, nameof(handler));
var attachmentEntry = archive.CreateEntry(ArchiveHelper.GetAttachmentPath(name));
using (var stream = attachmentEntry.Open())
{
await handler(stream);
}
writtenAttachments++;
}
public void WriteEvent(StoredEvent storedEvent)
{
Guard.NotNull(storedEvent, nameof(storedEvent));
var eventEntry = archive.CreateEntry(ArchiveHelper.GetEventPath(writtenEvents));
using (var stream = eventEntry.Open())
{
using (var textWriter = new StreamWriter(stream))
{
using (var jsonWriter = new JsonTextWriter(textWriter))
{
Serializer.Serialize(jsonWriter, storedEvent);
}
}
}
writtenEvents++;
}
}
}

79
src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs

@ -1,79 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class EventStreamWriter : DisposableObjectBase
{
private const int MaxItemsPerFolder = 1000;
private readonly ZipArchive archive;
private int writtenEvents;
private int writtenAttachments;
public EventStreamWriter(Stream stream)
{
archive = new ZipArchive(stream, ZipArchiveMode.Update, true);
}
public async Task WriteEventAsync(EventData eventData, Func<Stream, Task> attachment = null)
{
var eventObject =
new JObject(
new JProperty("type", eventData.Type),
new JProperty("payload", eventData.Payload),
new JProperty("metadata", eventData.Metadata));
var eventFolder = writtenEvents / MaxItemsPerFolder;
var eventPath = $"events/{eventFolder}/{writtenEvents}.json";
var eventEntry = archive.GetEntry(eventPath) ?? archive.CreateEntry(eventPath);
using (var stream = eventEntry.Open())
{
using (var textWriter = new StreamWriter(stream))
{
using (var jsonWriter = new JsonTextWriter(textWriter))
{
await eventObject.WriteToAsync(jsonWriter);
}
}
}
writtenEvents++;
if (attachment != null)
{
var attachmentFolder = writtenAttachments / MaxItemsPerFolder;
var attachmentPath = $"attachments/{attachmentFolder}/{writtenEvents}.blob";
var attachmentEntry = archive.GetEntry(attachmentPath) ?? archive.CreateEntry(attachmentPath);
using (var stream = attachmentEntry.Open())
{
await attachment(stream);
}
writtenAttachments++;
}
}
protected override void DisposeObject(bool disposing)
{
if (disposing)
{
archive.Dispose();
}
}
}
}

170
src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs

@ -0,0 +1,170 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using Newtonsoft.Json.Linq;
using Squidex.Infrastructure;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class GuidMapper
{
private static readonly int GuidLength = Guid.Empty.ToString().Length;
private readonly List<(JObject Source, string NewKey, string OldKey)> mappings = new List<(JObject Source, string NewKey, string OldKey)>();
private readonly Dictionary<Guid, Guid> oldToNewGuid = new Dictionary<Guid, Guid>();
private readonly Dictionary<Guid, Guid> newToOldGuid = new Dictionary<Guid, Guid>();
public Guid NewGuid(Guid oldGuid)
{
return oldToNewGuid.GetOrDefault(oldGuid);
}
public Guid OldGuid(Guid newGuid)
{
return newToOldGuid.GetOrDefault(newGuid);
}
public string NewGuidString(string key)
{
if (Guid.TryParse(key, out var guid))
{
return GenerateNewGuid(guid).ToString();
}
return null;
}
public JToken NewGuids(JToken jToken)
{
var result = NewGuidsCore(jToken);
if (mappings.Count > 0)
{
foreach (var mapping in mappings)
{
if (mapping.Source.TryGetValue(mapping.OldKey, out var value))
{
mapping.Source.Remove(mapping.OldKey);
mapping.Source[mapping.NewKey] = value;
}
}
mappings.Clear();
}
return result;
}
private JToken NewGuidsCore(JToken jToken)
{
switch (jToken.Type)
{
case JTokenType.String:
if (TryConvertString(jToken.ToString(), out var result))
{
return result;
}
break;
case JTokenType.Guid:
return GenerateNewGuid((Guid)jToken);
case JTokenType.Object:
NewGuidsCore((JObject)jToken);
break;
case JTokenType.Array:
NewGuidsCore((JArray)jToken);
break;
}
return jToken;
}
private void NewGuidsCore(JArray jArray)
{
for (var i = 0; i < jArray.Count; i++)
{
jArray[i] = NewGuidsCore(jArray[i]);
}
}
private void NewGuidsCore(JObject jObject)
{
foreach (var jProperty in jObject.Properties())
{
var newValue = NewGuidsCore(jProperty.Value);
if (!ReferenceEquals(newValue, jProperty.Value))
{
jProperty.Value = newValue;
}
if (TryConvertString(jProperty.Name, out var newKey))
{
mappings.Add((jObject, newKey, jProperty.Name));
}
}
}
private bool TryConvertString(string value, out string result)
{
return TryGenerateNewGuidString(value, out result) || TryGenerateNewNamedId(value, out result);
}
private bool TryGenerateNewGuidString(string value, out string result)
{
result = null;
if (value.Length == GuidLength)
{
if (Guid.TryParse(value, out var guid))
{
var newGuid = GenerateNewGuid(guid);
result = newGuid.ToString();
return true;
}
}
return false;
}
private bool TryGenerateNewNamedId(string value, out string result)
{
result = null;
if (value.Length > GuidLength && value[GuidLength] == ',')
{
if (Guid.TryParse(value.Substring(0, GuidLength), out var guid))
{
var newGuid = GenerateNewGuid(guid);
result = newGuid + value.Substring(GuidLength);
return true;
}
}
return false;
}
private Guid GenerateNewGuid(Guid oldGuid)
{
return oldToNewGuid.GetOrAdd(oldGuid, GuidGenerator);
}
private Guid GuidGenerator(Guid oldGuid)
{
var newGuid = Guid.NewGuid();
newToOldGuid[newGuid] = oldGuid;
return newGuid;
}
}
}

50
src/Squidex.Domain.Apps.Entities/Backup/Helpers/ArchiveHelper.cs

@ -0,0 +1,50 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
namespace Squidex.Domain.Apps.Entities.Backup.Archive
{
public static class ArchiveHelper
{
private const int MaxAttachmentFolders = 1000;
private const int MaxEventsPerFolder = 1000;
public static string GetAttachmentPath(string name)
{
name = name.ToLowerInvariant();
var attachmentFolder = SimpleHash(name) % MaxAttachmentFolders;
var attachmentPath = $"attachments/{attachmentFolder}/{name}";
return attachmentPath;
}
public static string GetEventPath(int index)
{
var eventFolder = index / MaxEventsPerFolder;
var eventPath = $"events/{eventFolder}/{index}.json";
return eventPath;
}
private static int SimpleHash(string value)
{
var hash = 17;
foreach (char c in value)
{
unchecked
{
hash = (hash * 23) + c.GetHashCode();
}
}
return Math.Abs(hash);
}
}
}

66
src/Squidex.Domain.Apps.Entities/Backup/Helpers/Downloader.cs

@ -0,0 +1,66 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
namespace Squidex.Domain.Apps.Entities.Backup.Helpers
{
public static class Downloader
{
public static async Task DownloadAsync(this IBackupArchiveLocation backupArchiveLocation, Uri url, Guid id)
{
HttpResponseMessage response = null;
try
{
using (var client = new HttpClient())
{
response = await client.GetAsync(url);
response.EnsureSuccessStatusCode();
using (var sourceStream = await response.Content.ReadAsStreamAsync())
{
using (var targetStream = await backupArchiveLocation.OpenStreamAsync(id))
{
await sourceStream.CopyToAsync(targetStream);
}
}
}
}
catch (HttpRequestException ex)
{
throw new BackupRestoreException($"Cannot download the archive. Got status code: {response?.StatusCode}.", ex);
}
}
public static async Task<BackupReader> OpenArchiveAsync(this IBackupArchiveLocation backupArchiveLocation, Guid id)
{
Stream stream = null;
try
{
stream = await backupArchiveLocation.OpenStreamAsync(id);
return new BackupReader(stream);
}
catch (IOException)
{
stream?.Dispose();
throw new BackupRestoreException("The backup archive is correupt and cannot be opened.");
}
catch (Exception)
{
stream?.Dispose();
throw;
}
}
}
}

62
src/Squidex.Domain.Apps.Entities/Backup/Helpers/Safe.cs

@ -0,0 +1,62 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Threading.Tasks;
using Squidex.Infrastructure.Assets;
using Squidex.Infrastructure.Log;
namespace Squidex.Domain.Apps.Entities.Backup.Helpers
{
public static class Safe
{
public static async Task DeleteAsync(IBackupArchiveLocation backupArchiveLocation, Guid id, ISemanticLog log)
{
try
{
await backupArchiveLocation.DeleteArchiveAsync(id);
}
catch (Exception ex)
{
log.LogError(ex, w => w
.WriteProperty("action", "deleteArchive")
.WriteProperty("status", "failed")
.WriteProperty("operationId", id.ToString()));
}
}
public static async Task DeleteAsync(IAssetStore assetStore, Guid id, ISemanticLog log)
{
try
{
await assetStore.DeleteAsync(id.ToString(), 0, null);
}
catch (Exception ex)
{
log.LogError(ex, w => w
.WriteProperty("action", "deleteBackup")
.WriteProperty("status", "failed")
.WriteProperty("operationId", id.ToString()));
}
}
public static async Task CleanupRestoreAsync(BackupHandler handler, Guid appId, Guid id, ISemanticLog log)
{
try
{
await handler.CleanupRestoreAsync(appId);
}
catch (Exception ex)
{
log.LogError(ex, w => w
.WriteProperty("action", "cleanupRestore")
.WriteProperty("status", "failed")
.WriteProperty("operationId", id.ToString()));
}
}
}
}

2
src/Squidex.Domain.Apps.Entities/Backup/IBackupJob.cs

@ -22,6 +22,6 @@ namespace Squidex.Domain.Apps.Entities.Backup
int HandledAssets { get; }
bool IsFailed { get; }
JobStatus Status { get; }
}
}

21
src/Squidex.Domain.Apps.Entities/Backup/IRestoreGrain.cs

@ -0,0 +1,21 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Threading.Tasks;
using Orleans;
using Squidex.Infrastructure.Orleans;
namespace Squidex.Domain.Apps.Entities.Backup
{
public interface IRestoreGrain : IGrainWithStringKey
{
Task RestoreAsync(Uri url, string newAppName = null);
Task<J<IRestoreJob>> GetJobAsync();
}
}

26
src/Squidex.Domain.Apps.Entities/Backup/IRestoreJob.cs

@ -0,0 +1,26 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using NodaTime;
namespace Squidex.Domain.Apps.Entities.Backup
{
public interface IRestoreJob
{
Uri Url { get; }
Instant Started { get; }
Instant? Stopped { get; }
List<string> Log { get; }
JobStatus Status { get; }
}
}

17
src/Squidex.Domain.Apps.Entities/Backup/JobStatus.cs

@ -0,0 +1,17 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
namespace Squidex.Domain.Apps.Entities.Backup
{
public enum JobStatus
{
Created,
Started,
Completed,
Failed
}
}

335
src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs

@ -0,0 +1,335 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using NodaTime;
using Orleans;
using Squidex.Domain.Apps.Entities.Backup.Helpers;
using Squidex.Domain.Apps.Entities.Backup.State;
using Squidex.Domain.Apps.Events;
using Squidex.Domain.Apps.Events.Apps;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Assets;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.Log;
using Squidex.Infrastructure.Orleans;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Backup
{
public sealed class RestoreGrain : GrainOfString, IRestoreGrain
{
private readonly IAssetStore assetStore;
private readonly IBackupArchiveLocation backupArchiveLocation;
private readonly IClock clock;
private readonly IEnumerable<BackupHandler> handlers;
private readonly IEventStore eventStore;
private readonly IEventDataFormatter eventDataFormatter;
private readonly IGrainFactory grainFactory;
private readonly ISemanticLog log;
private readonly IStreamNameResolver streamNameResolver;
private readonly IStore<string> store;
private RefToken actor;
private RestoreState state = new RestoreState();
private IPersistence<RestoreState> persistence;
private RestoreStateJob CurrentJob
{
get { return state.Job; }
}
public RestoreGrain(
IAssetStore assetStore,
IBackupArchiveLocation backupArchiveLocation,
IClock clock,
IEventStore eventStore,
IEventDataFormatter eventDataFormatter,
IGrainFactory grainFactory,
IEnumerable<BackupHandler> handlers,
ISemanticLog log,
IStreamNameResolver streamNameResolver,
IStore<string> store)
{
Guard.NotNull(assetStore, nameof(assetStore));
Guard.NotNull(backupArchiveLocation, nameof(backupArchiveLocation));
Guard.NotNull(clock, nameof(clock));
Guard.NotNull(eventStore, nameof(eventStore));
Guard.NotNull(eventDataFormatter, nameof(eventDataFormatter));
Guard.NotNull(grainFactory, nameof(grainFactory));
Guard.NotNull(handlers, nameof(handlers));
Guard.NotNull(store, nameof(store));
Guard.NotNull(streamNameResolver, nameof(streamNameResolver));
Guard.NotNull(log, nameof(log));
this.assetStore = assetStore;
this.backupArchiveLocation = backupArchiveLocation;
this.clock = clock;
this.eventStore = eventStore;
this.eventDataFormatter = eventDataFormatter;
this.grainFactory = grainFactory;
this.handlers = handlers;
this.store = store;
this.streamNameResolver = streamNameResolver;
this.log = log;
}
public override async Task OnActivateAsync(string key)
{
actor = new RefToken(RefTokenType.Subject, key);
persistence = store.WithSnapshots<RestoreState, string>(GetType(), key, s => state = s);
await ReadAsync();
RecoverAfterRestart();
}
private void RecoverAfterRestart()
{
RecoverAfterRestartAsync().Forget();
}
private async Task RecoverAfterRestartAsync()
{
if (CurrentJob?.Status == JobStatus.Started)
{
Log("Failed due application restart");
CurrentJob.Status = JobStatus.Failed;
await CleanupAsync();
await WriteAsync();
}
}
public Task RestoreAsync(Uri url, string newAppName)
{
Guard.NotNull(url, nameof(url));
if (newAppName != null)
{
Guard.ValidSlug(newAppName, nameof(newAppName));
}
if (CurrentJob?.Status == JobStatus.Started)
{
throw new DomainException("A restore operation is already running.");
}
state.Job = new RestoreStateJob
{
Id = Guid.NewGuid(),
NewAppName = newAppName,
Started = clock.GetCurrentInstant(),
Status = JobStatus.Started,
Url = url
};
Process();
return TaskHelper.Done;
}
private void Process()
{
ProcessAsync().Forget();
}
private async Task ProcessAsync()
{
using (Profiler.StartSession())
{
try
{
Log("Started. The restore process has the following steps:");
Log(" * Download backup");
Log(" * Restore events and attachments.");
Log(" * Restore all objects like app, schemas and contents");
Log(" * Complete the restore operation for all objects");
log.LogInformation(w => w
.WriteProperty("action", "restore")
.WriteProperty("status", "started")
.WriteProperty("operationId", CurrentJob.Id.ToString())
.WriteProperty("url", CurrentJob.Url.ToString()));
using (Profiler.Trace("Download"))
{
await DownloadAsync();
}
using (var reader = await backupArchiveLocation.OpenArchiveAsync(CurrentJob.Id))
{
using (Profiler.Trace("ReadEvents"))
{
await ReadEventsAsync(reader);
}
foreach (var handler in handlers)
{
using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.RestoreAsync)))
{
await handler.RestoreAsync(CurrentJob.AppId, reader);
}
Log($"Restored {handler.Name}");
}
foreach (var handler in handlers)
{
using (Profiler.TraceMethod(handler.GetType(), nameof(BackupHandler.CompleteRestoreAsync)))
{
await handler.CompleteRestoreAsync(CurrentJob.AppId, reader);
}
Log($"Completed {handler.Name}");
}
}
CurrentJob.Status = JobStatus.Completed;
Log("Completed, Yeah!");
log.LogInformation(w =>
{
w.WriteProperty("action", "restore");
w.WriteProperty("status", "completed");
w.WriteProperty("operationId", CurrentJob.Id.ToString());
w.WriteProperty("url", CurrentJob.Url.ToString());
Profiler.Session?.Write(w);
});
}
catch (Exception ex)
{
if (ex is BackupRestoreException backupException)
{
Log(backupException.Message);
}
else
{
Log("Failed with internal error");
}
await CleanupAsync(ex);
CurrentJob.Status = JobStatus.Failed;
log.LogError(ex, w =>
{
w.WriteProperty("action", "retore");
w.WriteProperty("status", "failed");
w.WriteProperty("operationId", CurrentJob.Id.ToString());
w.WriteProperty("url", CurrentJob.Url.ToString());
Profiler.Session?.Write(w);
});
}
finally
{
CurrentJob.Stopped = clock.GetCurrentInstant();
await WriteAsync();
}
}
}
private async Task CleanupAsync(Exception exception = null)
{
await Safe.DeleteAsync(backupArchiveLocation, CurrentJob.Id, log);
if (CurrentJob.AppId != Guid.Empty)
{
foreach (var handler in handlers)
{
await Safe.CleanupRestoreAsync(handler, CurrentJob.AppId, CurrentJob.Id, log);
}
}
}
private async Task DownloadAsync()
{
Log("Downloading Backup");
await backupArchiveLocation.DownloadAsync(CurrentJob.Url, CurrentJob.Id);
Log("Downloaded Backup");
}
private async Task ReadEventsAsync(BackupReader reader)
{
await reader.ReadEventsAsync(streamNameResolver, async (storedEvent) =>
{
var @event = eventDataFormatter.Parse(storedEvent.Data);
if (@event.Payload is SquidexEvent squidexEvent)
{
squidexEvent.Actor = actor;
}
if (@event.Payload is AppCreated appCreated)
{
CurrentJob.AppId = appCreated.AppId.Id;
if (!string.IsNullOrWhiteSpace(CurrentJob.NewAppName))
{
appCreated.Name = CurrentJob.NewAppName;
}
}
if (@event.Payload is AppEvent appEvent && !string.IsNullOrWhiteSpace(CurrentJob.NewAppName))
{
appEvent.AppId = new NamedId<Guid>(appEvent.AppId.Id, CurrentJob.NewAppName);
}
foreach (var handler in handlers)
{
await handler.RestoreEventAsync(@event, CurrentJob.AppId, reader, actor);
}
var eventData = eventDataFormatter.ToEventData(@event, @event.Headers.CommitId());
var eventCommit = new List<EventData> { eventData };
await eventStore.AppendAsync(Guid.NewGuid(), storedEvent.StreamName, eventCommit);
Log($"Read {reader.ReadEvents} events and {reader.ReadAttachments} attachments.", true);
});
Log("Reading events completed.");
}
private void Log(string message, bool replace = false)
{
if (replace && CurrentJob.Log.Count > 0)
{
CurrentJob.Log[CurrentJob.Log.Count - 1] = $"{clock.GetCurrentInstant()}: {message}";
}
else
{
CurrentJob.Log.Add($"{clock.GetCurrentInstant()}: {message}");
}
}
private async Task ReadAsync()
{
await persistence.ReadAsync();
}
private async Task WriteAsync()
{
await persistence.WriteSnapshotAsync(state);
}
public Task<J<IRestoreJob>> GetJobAsync()
{
return Task.FromResult<J<IRestoreJob>>(CurrentJob);
}
}
}

2
src/Squidex.Domain.Apps.Entities/Backup/State/BackupStateJob.cs

@ -29,6 +29,6 @@ namespace Squidex.Domain.Apps.Entities.Backup.State
public int HandledAssets { get; set; }
[JsonProperty]
public bool IsFailed { get; set; }
public JobStatus Status { get; set; }
}
}

17
src/Squidex.Domain.Apps.Entities/Backup/State/RestoreState.cs

@ -0,0 +1,17 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Newtonsoft.Json;
namespace Squidex.Domain.Apps.Entities.Backup.State
{
public class RestoreState
{
[JsonProperty]
public RestoreStateJob Job { get; set; }
}
}

44
src/Squidex.Domain.Apps.Entities/Backup/State/RestoreStateJob.cs

@ -0,0 +1,44 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
using NodaTime;
namespace Squidex.Domain.Apps.Entities.Backup.State
{
public sealed class RestoreStateJob : IRestoreJob
{
[JsonProperty]
public string AppName { get; set; }
[JsonProperty]
public Guid Id { get; set; }
[JsonProperty]
public Guid AppId { get; set; }
[JsonProperty]
public Uri Url { get; set; }
[JsonProperty]
public string NewAppName { get; set; }
[JsonProperty]
public Instant Started { get; set; }
[JsonProperty]
public Instant? Stopped { get; set; }
[JsonProperty]
public List<string> Log { get; set; } = new List<string>();
[JsonProperty]
public JobStatus Status { get; set; }
}
}

4
src/Squidex.Domain.Apps.Entities/Backup/TempFolderBackupArchiveLocation.cs

@ -18,7 +18,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
{
var tempFile = GetTempFile(backupId);
return Task.FromResult<Stream>(new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite));
return Task.FromResult<Stream>(new FileStream(tempFile, FileMode.OpenOrCreate, FileAccess.ReadWrite));
}
public Task DeleteArchiveAsync(Guid backupId)
@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
private static string GetTempFile(Guid backupId)
{
return Path.Combine(Path.GetTempPath(), backupId.ToString());
return Path.Combine(Path.GetTempPath(), backupId.ToString() + ".zip");
}
}
}

54
src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs

@ -0,0 +1,54 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Domain.Apps.Entities.Contents.Repositories;
using Squidex.Domain.Apps.Entities.Contents.State;
using Squidex.Domain.Apps.Events.Contents;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Contents
{
public sealed class BackupContents : BackupHandlerWithStore
{
private readonly HashSet<Guid> contentIds = new HashSet<Guid>();
private readonly IContentRepository contentRepository;
public override string Name { get; } = "Contents";
public BackupContents(IStore<Guid> store, IContentRepository contentRepository)
: base(store)
{
Guard.NotNull(contentRepository, nameof(contentRepository));
this.contentRepository = contentRepository;
}
public override Task RestoreEventAsync(Envelope<IEvent> @event, Guid appId, BackupReader reader, RefToken actor)
{
switch (@event.Payload)
{
case ContentCreated contentCreated:
contentIds.Add(contentCreated.ContentId);
break;
}
return TaskHelper.Done;
}
public override Task RestoreAsync(Guid appId, BackupReader reader)
{
return RebuildManyAsync(contentIds, id => RebuildAsync<ContentState, ContentGrain>(id, (e, s) => s.Apply(e)));
}
}
}

2
src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs

@ -28,5 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Repositories
Task<IContentEntity> FindContentAsync(IAppEntity app, ISchemaEntity schema, Status[] status, Guid id);
Task QueryScheduledWithoutDataAsync(Instant now, Func<IContentEntity, Task> callback);
Task RemoveAsync(Guid appId);
}
}

2
src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs

@ -14,5 +14,7 @@ namespace Squidex.Domain.Apps.Entities.History.Repositories
public interface IHistoryEventRepository
{
Task<IReadOnlyList<IHistoryEventEntity>> QueryByChannelAsync(Guid appId, string channelPrefix, int count);
Task RemoveAsync(Guid appId);
}
}

65
src/Squidex.Domain.Apps.Entities/Rules/BackupRules.cs

@ -0,0 +1,65 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Orleans;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Domain.Apps.Entities.Rules.Indexes;
using Squidex.Domain.Apps.Entities.Rules.Repositories;
using Squidex.Domain.Apps.Entities.Rules.State;
using Squidex.Domain.Apps.Events.Rules;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Rules
{
public sealed class BackupRules : BackupHandlerWithStore
{
private readonly HashSet<Guid> ruleIds = new HashSet<Guid>();
private readonly IGrainFactory grainFactory;
private readonly IRuleEventRepository ruleEventRepository;
public override string Name { get; } = "Rules";
public BackupRules(IStore<Guid> store, IGrainFactory grainFactory, IRuleEventRepository ruleEventRepository)
: base(store)
{
Guard.NotNull(grainFactory, nameof(grainFactory));
Guard.NotNull(ruleEventRepository, nameof(ruleEventRepository));
this.grainFactory = grainFactory;
this.ruleEventRepository = ruleEventRepository;
}
public override Task RestoreEventAsync(Envelope<IEvent> @event, Guid appId, BackupReader reader, RefToken actor)
{
switch (@event.Payload)
{
case RuleCreated ruleCreated:
ruleIds.Add(ruleCreated.RuleId);
break;
case RuleDeleted ruleDeleted:
ruleIds.Remove(ruleDeleted.RuleId);
break;
}
return TaskHelper.Done;
}
public async override Task RestoreAsync(Guid appId, BackupReader reader)
{
await RebuildManyAsync(ruleIds, id => RebuildAsync<RuleState, RuleGrain>(id, (e, s) => s.Apply(e)));
await grainFactory.GetGrain<IRulesByAppIndex>(appId).RebuildAsync(ruleIds);
}
}
}

4
src/Squidex.Domain.Apps.Entities/Rules/Indexes/IRulesByAppIndex.cs

@ -10,7 +10,7 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Orleans;
namespace Squidex.Domain.Apps.Entities.Rules
namespace Squidex.Domain.Apps.Entities.Rules.Indexes
{
public interface IRulesByAppIndex : IGrainWithGuidKey
{
@ -20,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Rules
Task RebuildAsync(HashSet<Guid> rules);
Task ClearAsync();
Task<List<Guid>> GetRuleIdsAsync();
}
}

7
src/Squidex.Domain.Apps.Entities/Rules/Indexes/RulesByAppIndexGrain.cs

@ -44,6 +44,13 @@ namespace Squidex.Domain.Apps.Entities.Rules.Indexes
return persistence.ReadAsync();
}
public Task ClearAsync()
{
state = new State();
return persistence.DeleteAsync();
}
public Task RebuildAsync(HashSet<Guid> rules)
{
state = new State { Rules = rules };

2
src/Squidex.Domain.Apps.Entities/Rules/Repositories/IRuleEventRepository.cs

@ -25,6 +25,8 @@ namespace Squidex.Domain.Apps.Entities.Rules.Repositories
Task QueryPendingAsync(Instant now, Func<IRuleEventEntity, Task> callback, CancellationToken ct = default(CancellationToken));
Task RemoveAsync(Guid appId);
Task<int> CountByAppAsync(Guid appId);
Task<IReadOnlyList<IRuleEventEntity>> QueryByAppAsync(Guid appId, int skip = 0, int take = 20);

65
src/Squidex.Domain.Apps.Entities/Schemas/BackupSchemas.cs

@ -0,0 +1,65 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Orleans;
using Squidex.Domain.Apps.Core.Schemas;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Domain.Apps.Entities.Schemas.Indexes;
using Squidex.Domain.Apps.Entities.Schemas.State;
using Squidex.Domain.Apps.Events.Schemas;
using Squidex.Infrastructure;
using Squidex.Infrastructure.EventSourcing;
using Squidex.Infrastructure.States;
using Squidex.Infrastructure.Tasks;
namespace Squidex.Domain.Apps.Entities.Schemas
{
public sealed class BackupSchemas : BackupHandlerWithStore
{
private readonly HashSet<NamedId<Guid>> schemaIds = new HashSet<NamedId<Guid>>();
private readonly Dictionary<string, Guid> schemasByName = new Dictionary<string, Guid>();
private readonly FieldRegistry fieldRegistry;
private readonly IGrainFactory grainFactory;
public override string Name { get; } = "Schemas";
public BackupSchemas(IStore<Guid> store, FieldRegistry fieldRegistry, IGrainFactory grainFactory)
: base(store)
{
Guard.NotNull(fieldRegistry, nameof(fieldRegistry));
Guard.NotNull(grainFactory, nameof(grainFactory));
this.fieldRegistry = fieldRegistry;
this.grainFactory = grainFactory;
}
public override Task RestoreEventAsync(Envelope<IEvent> @event, Guid appId, BackupReader reader, RefToken actor)
{
switch (@event.Payload)
{
case SchemaCreated schemaCreated:
schemaIds.Add(schemaCreated.SchemaId);
schemasByName[schemaCreated.SchemaId.Name] = schemaCreated.SchemaId.Id;
break;
}
return TaskHelper.Done;
}
public async override Task RestoreAsync(Guid appId, BackupReader reader)
{
await RebuildManyAsync(schemaIds.Select(x => x.Id), id => RebuildAsync<SchemaState, SchemaGrain>(id, (e, s) => s.Apply(e, fieldRegistry)));
await grainFactory.GetGrain<ISchemasByAppIndex>(appId).RebuildAsync(schemasByName);
}
}
}

4
src/Squidex.Domain.Apps.Entities/Schemas/Indexes/ISchemasByAppIndex.cs

@ -10,7 +10,7 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Orleans;
namespace Squidex.Domain.Apps.Entities.Schemas
namespace Squidex.Domain.Apps.Entities.Schemas.Indexes
{
public interface ISchemasByAppIndex : IGrainWithGuidKey
{
@ -20,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Schemas
Task RebuildAsync(Dictionary<string, Guid> schemas);
Task ClearAsync();
Task<Guid> GetSchemaIdAsync(string name);
Task<List<Guid>> GetSchemaIdsAsync();

7
src/Squidex.Domain.Apps.Entities/Schemas/Indexes/SchemasByAppIndexGrain.cs

@ -44,6 +44,13 @@ namespace Squidex.Domain.Apps.Entities.Schemas.Indexes
return persistence.ReadAsync();
}
public Task ClearAsync()
{
state = new State();
return persistence.DeleteAsync();
}
public Task RebuildAsync(Dictionary<string, Guid> schemas)
{
state = new State { Schemas = schemas };

42
src/Squidex.Domain.Apps.Entities/Tags/GrainTagService.cs

@ -17,6 +17,11 @@ namespace Squidex.Domain.Apps.Entities.Tags
{
private readonly IGrainFactory grainFactory;
public string Name
{
get { return "Tags"; }
}
public GrainTagService(IGrainFactory grainFactory)
{
Guard.NotNull(grainFactory, nameof(grainFactory));
@ -24,31 +29,46 @@ namespace Squidex.Domain.Apps.Entities.Tags
this.grainFactory = grainFactory;
}
public Task<HashSet<string>> NormalizeTagsAsync(Guid appId, string category, HashSet<string> names, HashSet<string> ids)
public Task<HashSet<string>> NormalizeTagsAsync(Guid appId, string group, HashSet<string> names, HashSet<string> ids)
{
return GetGrain(appId, group).NormalizeTagsAsync(names, ids);
}
public Task<HashSet<string>> GetTagIdsAsync(Guid appId, string group, HashSet<string> names)
{
return GetGrain(appId, group).GetTagIdsAsync(names);
}
public Task<Dictionary<string, string>> DenormalizeTagsAsync(Guid appId, string group, HashSet<string> ids)
{
return GetGrain(appId, group).DenormalizeTagsAsync(ids);
}
public Task<Dictionary<string, int>> GetTagsAsync(Guid appId, string group)
{
return GetGrain(appId, category).NormalizeTagsAsync(names, ids);
return GetGrain(appId, group).GetTagsAsync();
}
public Task<HashSet<string>> GetTagIdsAsync(Guid appId, string category, HashSet<string> names)
public Task<TagSet> GetExportableTagsAsync(Guid appId, string group)
{
return GetGrain(appId, category).GetTagIdsAsync(names);
return GetGrain(appId, group).GetExportableTagsAsync();
}
public Task<Dictionary<string, string>> DenormalizeTagsAsync(Guid appId, string category, HashSet<string> ids)
public Task RebuildTagsAsync(Guid appId, string group, TagSet tags)
{
return GetGrain(appId, category).DenormalizeTagsAsync(ids);
return GetGrain(appId, group).RebuildAsync(tags);
}
public Task<Dictionary<string, int>> GetTagsAsync(Guid appId, string category)
public Task ClearAsync(Guid appId, string group)
{
return GetGrain(appId, category).GetTagsAsync();
return GetGrain(appId, group).ClearAsync();
}
private ITagGrain GetGrain(Guid appId, string category)
private ITagGrain GetGrain(Guid appId, string group)
{
Guard.NotNullOrEmpty(category, nameof(category));
Guard.NotNullOrEmpty(group, nameof(group));
return grainFactory.GetGrain<ITagGrain>($"{appId}_{category}");
return grainFactory.GetGrain<ITagGrain>($"{appId}_{group}");
}
}
}

6
src/Squidex.Domain.Apps.Entities/Tags/ITagGrain.cs

@ -20,5 +20,11 @@ namespace Squidex.Domain.Apps.Entities.Tags
Task<Dictionary<string, string>> DenormalizeTagsAsync(HashSet<string> ids);
Task<Dictionary<string, int>> GetTagsAsync();
Task<TagSet> GetExportableTagsAsync();
Task ClearAsync();
Task RebuildAsync(TagSet tags);
}
}

14
src/Squidex.Domain.Apps.Entities/Tags/ITagService.cs

@ -13,12 +13,18 @@ namespace Squidex.Domain.Apps.Entities.Tags
{
public interface ITagService
{
Task<HashSet<string>> NormalizeTagsAsync(Guid appId, string category, HashSet<string> names, HashSet<string> ids);
Task<HashSet<string>> NormalizeTagsAsync(Guid appId, string group, HashSet<string> names, HashSet<string> ids);
Task<HashSet<string>> GetTagIdsAsync(Guid appId, string category, HashSet<string> names);
Task<HashSet<string>> GetTagIdsAsync(Guid appId, string group, HashSet<string> names);
Task<Dictionary<string, string>> DenormalizeTagsAsync(Guid appId, string category, HashSet<string> ids);
Task<Dictionary<string, string>> DenormalizeTagsAsync(Guid appId, string group, HashSet<string> ids);
Task<Dictionary<string, int>> GetTagsAsync(Guid appId, string category);
Task<Dictionary<string, int>> GetTagsAsync(Guid appId, string group);
Task<TagSet> GetExportableTagsAsync(Guid appId, string group);
Task RebuildTagsAsync(Guid appId, string group, TagSet tags);
Task ClearAsync(Guid appId, string group);
}
}

16
src/Squidex.Domain.Apps.Entities/Tags/Tag.cs

@ -0,0 +1,16 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
namespace Squidex.Domain.Apps.Entities.Tags
{
public sealed class Tag
{
public string Name { get; set; }
public int Count { get; set; } = 1;
}
}

30
src/Squidex.Domain.Apps.Entities/Tags/TagGrain.cs

@ -24,14 +24,7 @@ namespace Squidex.Domain.Apps.Entities.Tags
[CollectionName("Index_Tags")]
public sealed class State
{
public Dictionary<string, TagInfo> Tags { get; set; } = new Dictionary<string, TagInfo>();
}
public sealed class TagInfo
{
public string Name { get; set; }
public int Count { get; set; } = 1;
public TagSet Tags { get; set; } = new TagSet();
}
public TagGrain(IStore<string> store)
@ -51,6 +44,20 @@ namespace Squidex.Domain.Apps.Entities.Tags
return persistence.ReadAsync();
}
public Task ClearAsync()
{
state = new State();
return persistence.DeleteAsync();
}
public Task RebuildAsync(TagSet tags)
{
state.Tags = tags;
return persistence.WriteSnapshotAsync(state);
}
public async Task<HashSet<string>> NormalizeTagsAsync(HashSet<string> names, HashSet<string> ids)
{
var result = new HashSet<string>();
@ -79,7 +86,7 @@ namespace Squidex.Domain.Apps.Entities.Tags
{
tagId = Guid.NewGuid().ToString();
state.Tags.Add(tagId, new TagInfo { Name = tagName });
state.Tags.Add(tagId, new Tag { Name = tagName });
}
result.Add(tagId);
@ -147,5 +154,10 @@ namespace Squidex.Domain.Apps.Entities.Tags
{
return Task.FromResult(state.Tags.Values.ToDictionary(x => x.Name, x => x.Count));
}
public Task<TagSet> GetExportableTagsAsync()
{
return Task.FromResult(state.Tags);
}
}
}

15
src/Squidex.Domain.Apps.Entities/Tags/TagSet.cs

@ -0,0 +1,15 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Collections.Generic;
namespace Squidex.Domain.Apps.Entities.Tags
{
public sealed class TagSet : Dictionary<string, Tag>
{
}
}

73
src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs

@ -57,82 +57,93 @@ namespace Squidex.Infrastructure.Assets
return new Uri(blobContainer.StorageUri.PrimaryUri, $"/{containerName}/{blobName}").ToString();
}
public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
public async Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
{
var blobName = GetObjectName(id, version, suffix);
var blobRef = blobContainer.GetBlobReference(blobName);
var targetName = GetObjectName(id, version, suffix);
var targetBlob = blobContainer.GetBlobReference(targetName);
var tempBlob = blobContainer.GetBlockBlobReference(name);
var sourceBlob = blobContainer.GetBlockBlobReference(sourceFileName);
try
{
await blobRef.StartCopyAsync(tempBlob.Uri, null, null, null, null, ct);
await targetBlob.StartCopyAsync(sourceBlob.Uri, null, AccessCondition.GenerateIfNotExistsCondition(), null, null, ct);
while (blobRef.CopyState.Status == CopyStatus.Pending)
while (targetBlob.CopyState.Status == CopyStatus.Pending)
{
ct.ThrowIfCancellationRequested();
await Task.Delay(50);
await blobRef.FetchAttributesAsync(null, null, null, ct);
await targetBlob.FetchAttributesAsync(null, null, null, ct);
}
if (blobRef.CopyState.Status != CopyStatus.Success)
if (targetBlob.CopyState.Status != CopyStatus.Success)
{
throw new StorageException($"Copy of temporary file failed: {blobRef.CopyState.Status}");
throw new StorageException($"Copy of temporary file failed: {targetBlob.CopyState.Status}");
}
}
catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 409)
{
throw new AssetAlreadyExistsException(targetName);
}
catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404)
{
throw new AssetNotFoundException($"Asset {name} not found.", ex);
throw new AssetNotFoundException(sourceFileName, ex);
}
}
public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
var blobName = GetObjectName(id, version, suffix);
var blobRef = blobContainer.GetBlockBlobReference(blobName);
var blob = blobContainer.GetBlockBlobReference(GetObjectName(id, version, suffix));
try
{
await blobRef.DownloadToStreamAsync(stream, null, null, null, ct);
await blob.DownloadToStreamAsync(stream, null, null, null, ct);
}
catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404)
{
throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
throw new AssetNotFoundException($"Id={id}, Version={version}", ex);
}
}
public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
var blobName = GetObjectName(id, version, suffix);
var blobRef = blobContainer.GetBlockBlobReference(blobName);
blobRef.Metadata[AssetVersion] = version.ToString();
blobRef.Metadata[AssetId] = id;
return UploadCoreAsync(GetObjectName(id, version, suffix), stream, ct);
}
await blobRef.UploadFromStreamAsync(stream, null, null, null, ct);
await blobRef.SetMetadataAsync();
public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken))
{
return UploadCoreAsync(fileName, stream, ct);
}
public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken))
public Task DeleteAsync(string id, long version, string suffix)
{
var tempBlob = blobContainer.GetBlockBlobReference(name);
return DeleteCoreAsync(GetObjectName(id, version, suffix));
}
return tempBlob.UploadFromStreamAsync(stream, null, null, null, ct);
public Task DeleteAsync(string fileName)
{
return DeleteCoreAsync(fileName);
}
public Task DeleteAsync(string name)
private Task DeleteCoreAsync(string blobName)
{
var tempBlob = blobContainer.GetBlockBlobReference(name);
var blob = blobContainer.GetBlockBlobReference(blobName);
return tempBlob.DeleteIfExistsAsync();
return blob.DeleteIfExistsAsync();
}
public Task DeleteAsync(string id, long version, string suffix)
private async Task UploadCoreAsync(string blobName, Stream stream, CancellationToken ct)
{
try
{
var tempBlob = blobContainer.GetBlockBlobReference(GetObjectName(id, version, suffix));
var tempBlob = blobContainer.GetBlockBlobReference(blobName);
return tempBlob.DeleteIfExistsAsync();
await tempBlob.UploadFromStreamAsync(stream, AccessCondition.GenerateIfNotExistsCondition(), null, null, ct);
}
catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 409)
{
throw new AssetAlreadyExistsException(blobName);
}
}
private string GetObjectName(string id, long version, string suffix)

1
src/Squidex.Infrastructure.GetEventStore/EventSourcing/Formatter.cs

@ -24,6 +24,7 @@ namespace Squidex.Infrastructure.EventSourcing
var eventData = new EventData { Type = @event.EventType, Payload = body, Metadata = meta };
return new StoredEvent(
@event.EventStreamId,
resolvedEvent.OriginalEventNumber.ToString(),
resolvedEvent.Event.EventNumber,
eventData);

10
src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStore.cs

@ -118,6 +118,11 @@ namespace Squidex.Infrastructure.EventSourcing
}
}
public Task DeleteStreamAsync(string streamName)
{
return connection.DeleteStreamAsync(streamName, ExpectedVersion.Any);
}
public Task AppendAsync(Guid commitId, string streamName, ICollection<EventData> events)
{
return AppendEventsInternalAsync(streamName, EtagVersion.Any, events);
@ -163,6 +168,11 @@ namespace Squidex.Infrastructure.EventSourcing
}
}
public Task DeleteManyAsync(string property, object value)
{
throw new NotSupportedException();
}
private string GetStreamName(string streamName)
{
return $"{prefix}-{streamName}";

66
src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs

@ -18,6 +18,8 @@ namespace Squidex.Infrastructure.Assets
{
public sealed class GoogleCloudAssetStore : IAssetStore, IInitializable
{
private static readonly UploadObjectOptions IfNotExists = new UploadObjectOptions { IfGenerationMatch = 0 };
private static readonly CopyObjectOptions IfNotExistsCopy = new CopyObjectOptions { IfGenerationMatch = 0 };
private readonly string bucketName;
private StorageClient storageClient;
@ -49,29 +51,21 @@ namespace Squidex.Infrastructure.Assets
return $"https://storage.cloud.google.com/{bucketName}/{objectName}";
}
public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken))
{
return storageClient.UploadObjectAsync(bucketName, name, "application/octet-stream", stream, cancellationToken: ct);
}
public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
var objectName = GetObjectName(id, version, suffix);
await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, cancellationToken: ct);
}
public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
public async Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
{
var objectName = GetObjectName(id, version, suffix);
try
{
await storageClient.CopyObjectAsync(bucketName, name, bucketName, objectName, cancellationToken: ct);
await storageClient.CopyObjectAsync(bucketName, sourceFileName, bucketName, objectName, IfNotExistsCopy, ct);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound)
{
throw new AssetNotFoundException($"Asset {name} not found.", ex);
throw new AssetNotFoundException(sourceFileName, ex);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.PreconditionFailed)
{
throw new AssetAlreadyExistsException(objectName);
}
}
@ -85,37 +79,51 @@ namespace Squidex.Infrastructure.Assets
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound)
{
throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
throw new AssetNotFoundException($"Id={id}, Version={version}", ex);
}
}
public async Task DeleteAsync(string name)
public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
try
{
await storageClient.DeleteObjectAsync(bucketName, name);
return UploadCoreAsync(GetObjectName(id, version, suffix), stream, ct);
}
catch (GoogleApiException ex)
{
if (ex.HttpStatusCode != HttpStatusCode.NotFound)
public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken))
{
throw;
return UploadCoreAsync(fileName, stream, ct);
}
public Task DeleteAsync(string id, long version, string suffix)
{
return DeleteCoreAsync(GetObjectName(id, version, suffix));
}
public Task DeleteAsync(string fileName)
{
return DeleteCoreAsync(fileName);
}
public async Task DeleteAsync(string id, long version, string suffix)
private async Task UploadCoreAsync(string objectName, Stream stream, CancellationToken ct)
{
try
{
await storageClient.DeleteObjectAsync(bucketName, GetObjectName(id, version, suffix));
await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, IfNotExists, ct);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.PreconditionFailed)
{
throw new AssetAlreadyExistsException(objectName);
}
}
catch (GoogleApiException ex)
private async Task DeleteCoreAsync(string objectName)
{
if (ex.HttpStatusCode != HttpStatusCode.NotFound)
try
{
throw;
await storageClient.DeleteObjectAsync(bucketName, objectName);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound)
{
return;
}
}

40
src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs

@ -9,6 +9,7 @@ using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
@ -43,20 +44,20 @@ namespace Squidex.Infrastructure.Assets
return "UNSUPPORTED";
}
public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
public async Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
{
try
{
var target = GetFileName(id, version, suffix);
using (var readStream = await bucket.OpenDownloadStreamAsync(name, cancellationToken: ct))
using (var readStream = await bucket.OpenDownloadStreamAsync(sourceFileName, cancellationToken: ct))
{
await bucket.UploadFromStreamAsync(target, target, readStream, cancellationToken: ct);
await UploadFileCoreAsync(target, readStream, ct);
}
}
catch (GridFSFileNotFoundException ex)
{
throw new AssetNotFoundException($"Asset {name} not found.", ex);
throw new AssetNotFoundException(sourceFileName, ex);
}
}
@ -73,13 +74,8 @@ namespace Squidex.Infrastructure.Assets
}
catch (GridFSFileNotFoundException ex)
{
throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
}
throw new AssetNotFoundException($"Id={id}, Version={version}", ex);
}
public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken))
{
return UploadFileCoreAsync(name, stream, ct);
}
public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
@ -87,9 +83,9 @@ namespace Squidex.Infrastructure.Assets
return UploadFileCoreAsync(GetFileName(id, version, suffix), stream, ct);
}
public Task DeleteAsync(string name)
public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken))
{
return DeleteCoreAsync(name);
return UploadFileCoreAsync(fileName, stream, ct);
}
public Task DeleteAsync(string id, long version, string suffix)
@ -97,6 +93,11 @@ namespace Squidex.Infrastructure.Assets
return DeleteCoreAsync(GetFileName(id, version, suffix));
}
public Task DeleteAsync(string fileName)
{
return DeleteCoreAsync(fileName);
}
private async Task DeleteCoreAsync(string id)
{
try
@ -109,9 +110,20 @@ namespace Squidex.Infrastructure.Assets
}
}
private Task UploadFileCoreAsync(string id, Stream stream, CancellationToken ct = default(CancellationToken))
private async Task UploadFileCoreAsync(string id, Stream stream, CancellationToken ct = default(CancellationToken))
{
return bucket.UploadFromStreamAsync(id, id, stream, cancellationToken: ct);
try
{
await bucket.UploadFromStreamAsync(id, id, stream, cancellationToken: ct);
}
catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey)
{
throw new AssetAlreadyExistsException(id);
}
catch (MongoBulkWriteException<BsonDocument> ex) when (ex.WriteErrors.Any(x => x.Category == ServerErrorCategory.DuplicateKey))
{
throw new AssetAlreadyExistsException(id);
}
}
private static string GetFileName(string id, long version, string suffix)

18
src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs

@ -60,7 +60,7 @@ namespace Squidex.Infrastructure.EventSourcing
var eventData = e.ToEventData();
var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length);
result.Add(new StoredEvent(eventToken, eventStreamOffset, eventData));
result.Add(new StoredEvent(streamName, eventToken, eventStreamOffset, eventData));
}
}
}
@ -111,7 +111,7 @@ namespace Squidex.Infrastructure.EventSourcing
var eventData = e.ToEventData();
var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length);
await callback(new StoredEvent(eventToken, eventStreamOffset, eventData));
await callback(new StoredEvent(commit.EventStream, eventToken, eventStreamOffset, eventData));
commitOffset++;
}
@ -124,8 +124,8 @@ namespace Squidex.Infrastructure.EventSourcing
{
var filters = new List<FilterDefinition<MongoEventCommit>>();
AddPositionFilter(streamPosition, filters);
AddPropertyFitler(property, value, filters);
FilterByPosition(streamPosition, filters);
FilterByProperty(property, value, filters);
return Filter.And(filters);
}
@ -134,18 +134,18 @@ namespace Squidex.Infrastructure.EventSourcing
{
var filters = new List<FilterDefinition<MongoEventCommit>>();
AddPositionFilter(streamPosition, filters);
AddStreamFilter(streamFilter, filters);
FilterByPosition(streamPosition, filters);
FilterByStream(streamFilter, filters);
return Filter.And(filters);
}
private static void AddPropertyFitler(string property, object value, List<FilterDefinition<MongoEventCommit>> filters)
private static void FilterByProperty(string property, object value, List<FilterDefinition<MongoEventCommit>> filters)
{
filters.Add(Filter.Eq(CreateIndexPath(property), value));
}
private static void AddStreamFilter(string streamFilter, List<FilterDefinition<MongoEventCommit>> filters)
private static void FilterByStream(string streamFilter, List<FilterDefinition<MongoEventCommit>> filters)
{
if (!string.IsNullOrWhiteSpace(streamFilter) && !string.Equals(streamFilter, ".*", StringComparison.OrdinalIgnoreCase))
{
@ -160,7 +160,7 @@ namespace Squidex.Infrastructure.EventSourcing
}
}
private static void AddPositionFilter(StreamPosition streamPosition, List<FilterDefinition<MongoEventCommit>> filters)
private static void FilterByPosition(StreamPosition streamPosition, List<FilterDefinition<MongoEventCommit>> filters)
{
if (streamPosition.IsEndOfCommit)
{

10
src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs

@ -20,6 +20,16 @@ namespace Squidex.Infrastructure.EventSourcing
private const int MaxWriteAttempts = 20;
private static readonly BsonTimestamp EmptyTimestamp = new BsonTimestamp(0);
public Task DeleteStreamAsync(string streamName)
{
return Collection.DeleteManyAsync(x => x.EventStream == streamName);
}
public Task DeleteManyAsync(string property, object value)
{
return Collection.DeleteManyAsync(Filter.Eq(CreateIndexPath(property), value));
}
public Task AppendAsync(Guid commitId, string streamName, ICollection<EventData> events)
{
return AppendAsync(commitId, streamName, EtagVersion.Any, events);

8
src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs

@ -68,5 +68,13 @@ namespace Squidex.Infrastructure.States
await Collection.Find(new BsonDocument()).ForEachAsync(x => callback(x.Doc, x.Version));
}
}
public async Task RemoveAsync(TKey key)
{
using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
{
await Collection.DeleteOneAsync(x => x.Id.Equals(key));
}
}
}
}

38
src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs

@ -0,0 +1,38 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschränkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Runtime.Serialization;
namespace Squidex.Infrastructure.Assets
{
[Serializable]
public class AssetAlreadyExistsException : Exception
{
public AssetAlreadyExistsException(string fileName)
: base(FormatMessage(fileName))
{
}
public AssetAlreadyExistsException(string fileName, Exception inner)
: base(FormatMessage(fileName), inner)
{
}
protected AssetAlreadyExistsException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
private static string FormatMessage(string fileName)
{
Guard.NotNullOrEmpty(fileName, nameof(fileName));
return $"An asset with name '{fileName}' already not exists.";
}
}
}

17
src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs

@ -13,23 +13,26 @@ namespace Squidex.Infrastructure.Assets
[Serializable]
public class AssetNotFoundException : Exception
{
public AssetNotFoundException()
public AssetNotFoundException(string fileName)
: base(FormatMessage(fileName))
{
}
public AssetNotFoundException(string message)
: base(message)
public AssetNotFoundException(string fileName, Exception inner)
: base(FormatMessage(fileName), inner)
{
}
public AssetNotFoundException(string message, Exception inner)
: base(message, inner)
protected AssetNotFoundException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
protected AssetNotFoundException(SerializationInfo info, StreamingContext context)
: base(info, context)
private static string FormatMessage(string fileName)
{
Guard.NotNullOrEmpty(fileName, nameof(fileName));
return $"An asset with name '{fileName}' does not exist.";
}
}
}

78
src/Squidex.Infrastructure/Assets/FolderAssetStore.cs

@ -59,26 +59,6 @@ namespace Squidex.Infrastructure.Assets
return file.FullName;
}
public async Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken))
{
var file = GetFile(name);
using (var fileStream = file.OpenWrite())
{
await stream.CopyToAsync(fileStream, BufferSize, ct);
}
}
public async Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
var file = GetFile(id, version, suffix);
using (var fileStream = file.OpenWrite())
{
await stream.CopyToAsync(fileStream, BufferSize, ct);
}
}
public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
var file = GetFile(id, version, suffix);
@ -92,44 +72,74 @@ namespace Squidex.Infrastructure.Assets
}
catch (FileNotFoundException ex)
{
throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
throw new AssetNotFoundException($"Id={id}, Version={version}", ex);
}
}
public Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
public Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
{
var targetFile = GetFile(id, version, suffix);
try
{
var file = GetFile(name);
var file = GetFile(sourceFileName);
file.CopyTo(GetPath(id, version, suffix));
file.CopyTo(targetFile.FullName);
return TaskHelper.Done;
}
catch (IOException) when (targetFile.Exists)
{
throw new AssetAlreadyExistsException(targetFile.Name);
}
catch (FileNotFoundException ex)
{
throw new AssetNotFoundException($"Asset {name} not found.", ex);
throw new AssetNotFoundException(sourceFileName, ex);
}
}
public Task DeleteAsync(string id, long version, string suffix)
public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
{
var file = GetFile(id, version, suffix);
return UploadCoreAsync(GetFile(id, version, suffix), stream, ct);
}
file.Delete();
public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken))
{
return UploadCoreAsync(GetFile(fileName), stream, ct);
}
return TaskHelper.Done;
public Task DeleteAsync(string id, long version, string suffix)
{
return DeleteFileCoreAsync(GetFile(id, version, suffix));
}
public Task DeleteAsync(string name)
public Task DeleteAsync(string fileName)
{
var file = GetFile(name);
return DeleteFileCoreAsync(GetFile(fileName));
}
private static Task DeleteFileCoreAsync(FileInfo file)
{
file.Delete();
return TaskHelper.Done;
}
private async Task UploadCoreAsync(FileInfo file, Stream stream, CancellationToken ct)
{
try
{
using (var fileStream = file.Open(FileMode.CreateNew, FileAccess.Write))
{
await stream.CopyToAsync(fileStream, BufferSize, ct);
}
}
catch (IOException) when (file.Exists)
{
throw new AssetAlreadyExistsException(file.Name);
}
}
private FileInfo GetFile(string id, long version, string suffix)
{
Guard.NotNullOrEmpty(id, nameof(id));
@ -137,11 +147,11 @@ namespace Squidex.Infrastructure.Assets
return GetFile(GetPath(id, version, suffix));
}
private FileInfo GetFile(string name)
private FileInfo GetFile(string fileName)
{
Guard.NotNullOrEmpty(name, nameof(name));
Guard.NotNullOrEmpty(fileName, nameof(fileName));
return new FileInfo(GetPath(name));
return new FileInfo(GetPath(fileName));
}
private string GetPath(string name)

6
src/Squidex.Infrastructure/Assets/IAssetStore.cs

@ -15,15 +15,15 @@ namespace Squidex.Infrastructure.Assets
{
string GenerateSourceUrl(string id, long version, string suffix);
Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken));
Task CopyAsync(string sourceFileName, string id, long version, string suffix, CancellationToken ct = default(CancellationToken));
Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken));
Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken));
Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default(CancellationToken));
Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken));
Task DeleteAsync(string name);
Task DeleteAsync(string fileName);
Task DeleteAsync(string id, long version, string suffix);
}

12
src/Squidex.Infrastructure/CollectionExtensions.cs

@ -167,6 +167,18 @@ namespace Squidex.Infrastructure
return result;
}
public static TValue GetOrAdd<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, TValue fallback)
{
if (!dictionary.TryGetValue(key, out var result))
{
result = fallback;
dictionary.Add(key, result);
}
return result;
}
public static TValue GetOrAdd<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> creator)
{
if (!dictionary.TryGetValue(key, out var result))

4
src/Squidex.Infrastructure/EventSourcing/IEventStore.cs

@ -26,6 +26,10 @@ namespace Squidex.Infrastructure.EventSourcing
Task AppendAsync(Guid commitId, string streamName, long expectedVersion, ICollection<EventData> events);
Task DeleteStreamAsync(string streamName);
Task DeleteManyAsync(string property, object value);
IEventSubscription CreateSubscription(IEventSubscriber subscriber, string streamFilter, string position = null);
}
}

7
src/Squidex.Infrastructure/EventSourcing/StoredEvent.cs

@ -9,14 +9,17 @@ namespace Squidex.Infrastructure.EventSourcing
{
public sealed class StoredEvent
{
public string StreamName { get; }
public string EventPosition { get; }
public long EventStreamNumber { get; }
public EventData Data { get; }
public StoredEvent(string eventPosition, long eventStreamNumber, EventData data)
public StoredEvent(string streamName, string eventPosition, long eventStreamNumber, EventData data)
{
Guard.NotNullOrEmpty(streamName, nameof(streamName));
Guard.NotNullOrEmpty(eventPosition, nameof(eventPosition));
Guard.NotNull(data, nameof(data));
@ -24,6 +27,8 @@ namespace Squidex.Infrastructure.EventSourcing
EventPosition = eventPosition;
EventStreamNumber = eventStreamNumber;
StreamName = streamName;
}
}
}

5
src/Squidex.Infrastructure/Log/Profiler.cs

@ -34,6 +34,11 @@ namespace Squidex.Infrastructure.Log
return Cleaner;
}
public static IDisposable TraceMethod(Type type, [CallerMemberName] string memberName = null)
{
return Trace($"{type.Name}/{memberName}");
}
public static IDisposable TraceMethod<T>([CallerMemberName] string memberName = null)
{
return Trace($"{typeof(T).Name}/{memberName}");

16
src/Squidex.Infrastructure/RefTokenType.cs

@ -0,0 +1,16 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
namespace Squidex.Infrastructure
{
public static class RefTokenType
{
public const string Subject = "subject";
public const string Client = "client";
}
}

23
src/Squidex.Infrastructure/States/DefaultStreamNameResolver.cs

@ -15,6 +15,9 @@ namespace Squidex.Infrastructure.States
public string GetStreamName(Type aggregateType, string id)
{
Guard.NotNullOrEmpty(id, nameof(id));
Guard.NotNull(aggregateType, nameof(aggregateType));
var typeName = char.ToLower(aggregateType.Name[0]) + aggregateType.Name.Substring(1);
foreach (var suffix in Suffixes)
@ -29,5 +32,25 @@ namespace Squidex.Infrastructure.States
return $"{typeName}-{id}";
}
public string WithNewId(string streamName, Func<string, string> idGenerator)
{
Guard.NotNullOrEmpty(streamName, nameof(streamName));
Guard.NotNull(idGenerator, nameof(idGenerator));
var positionOfDash = streamName.IndexOf('-');
if (positionOfDash >= 0)
{
var newId = idGenerator(streamName.Substring(positionOfDash + 1));
if (!string.IsNullOrWhiteSpace(newId))
{
streamName = $"{streamName.Substring(0, positionOfDash)}-{newId}";
}
}
return streamName;
}
}
}

2
src/Squidex.Infrastructure/States/IPersistence{TState}.cs

@ -15,6 +15,8 @@ namespace Squidex.Infrastructure.States
{
long Version { get; }
Task DeleteAsync();
Task WriteEventsAsync(IEnumerable<Envelope<IEvent>> @events);
Task WriteSnapshotAsync(TState state);

2
src/Squidex.Infrastructure/States/ISnapshotStore.cs

@ -18,6 +18,8 @@ namespace Squidex.Infrastructure.States
Task ClearAsync();
Task RemoveAsync(TKey key);
Task ReadAllAsync(Func<T, long, Task> callback);
}
}

2
src/Squidex.Infrastructure/States/IStore.cs

@ -20,7 +20,5 @@ namespace Squidex.Infrastructure.States
IPersistence<TState> WithSnapshotsAndEventSourcing<TState>(Type owner, TKey key, Func<TState, Task> applySnapshot, Func<Envelope<IEvent>, Task> applyEvent);
ISnapshotStore<TState, TKey> GetSnapshotStore<TState>();
Task ClearSnapshotsAsync<TState>();
}
}

2
src/Squidex.Infrastructure/States/IStreamNameResolver.cs

@ -12,5 +12,7 @@ namespace Squidex.Infrastructure.States
public interface IStreamNameResolver
{
string GetStreamName(Type aggregateType, string id);
string WithNewId(string streamName, Func<string, string> idGenerator);
}
}

13
src/Squidex.Infrastructure/States/Persistence{TSnapshot,TKey}.cs

@ -175,6 +175,19 @@ namespace Squidex.Infrastructure.States
UpdateVersion();
}
public async Task DeleteAsync()
{
if (UseEventSourcing())
{
await eventStore.DeleteStreamAsync(GetStreamName());
}
if (UseSnapshots())
{
await snapshotStore.RemoveAsync(ownerKey);
}
}
private EventData[] GetEventData(Envelope<IEvent>[] events, Guid commitId)
{
return @events.Select(x => eventDataFormatter.ToEventData(x, commitId, true)).ToArray();

5
src/Squidex.Infrastructure/States/Store.cs

@ -63,6 +63,11 @@ namespace Squidex.Infrastructure.States
return GetSnapshotStore<TState>().ClearAsync();
}
public Task RemoveSnapshotAsync<TState>(TKey key)
{
return GetSnapshotStore<TState>().RemoveAsync(key);
}
public ISnapshotStore<TState, TKey> GetSnapshotStore<TState>()
{
return (ISnapshotStore<TState, TKey>)services.GetService(typeof(ISnapshotStore<TState, TKey>));

17
src/Squidex.Infrastructure/States/StoreExtensions.cs

@ -58,5 +58,22 @@ namespace Squidex.Infrastructure.States
{
return store.WithSnapshotsAndEventSourcing(typeof(TOwner), key, applySnapshot.ToAsync(), applyEvent.ToAsync());
}
public static Task ClearSnapshotsAsync<TKey, TState>(this IStore<TKey> store)
{
return store.GetSnapshotStore<TState>().ClearAsync();
}
public static Task RemoveSnapshotAsync<TKey, TState>(this IStore<TKey> store, TKey key)
{
return store.GetSnapshotStore<TState>().RemoveAsync(key);
}
public static async Task<TState> GetSnapshotAsync<TKey, TState>(this IStore<TKey> store, TKey key)
{
var result = await store.GetSnapshotStore<TState>().ReadAsync(key);
return result.Value;
}
}
}

16
src/Squidex.Infrastructure/Tasks/TaskExtensions.cs

@ -6,14 +6,30 @@
// ==========================================================================
using System;
using System.Threading;
using System.Threading.Tasks;
namespace Squidex.Infrastructure.Tasks
{
public static class TaskExtensions
{
private static readonly Action<Task> IgnoreTaskContinuation = t => { var ignored = t.Exception; };
public static void Forget(this Task task)
{
if (task.IsCompleted)
{
var ignored = task.Exception;
}
else
{
task.ContinueWith(
IgnoreTaskContinuation,
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted |
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
}
public static Func<TInput, TOutput> ToDefault<TInput, TOutput>(this Action<TInput> action)

4
src/Squidex/Areas/Api/Controllers/Backups/Models/BackupJobDto.cs

@ -40,9 +40,9 @@ namespace Squidex.Areas.Api.Controllers.Backups.Models
public int HandledAssets { get; set; }
/// <summary>
/// Indicates if the job has failed.
/// The status of the operation.
/// </summary>
public bool IsFailed { get; set; }
public JobStatus Status { get; set; }
public static BackupJobDto FromBackup(IBackupJob backup)
{

51
src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreJobDto.cs

@ -0,0 +1,51 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using NodaTime;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Infrastructure.Reflection;
namespace Squidex.Areas.Api.Controllers.Backups.Models
{
public sealed class RestoreJobDto
{
/// <summary>
/// The uri to load from.
/// </summary>
[Required]
public Uri Url { get; set; }
/// <summary>
/// The status log.
/// </summary>
[Required]
public List<string> Log { get; set; }
/// <summary>
/// The time when the job has been started.
/// </summary>
public Instant Started { get; set; }
/// <summary>
/// The time when the job has been stopped.
/// </summary>
public Instant? Stopped { get; set; }
/// <summary>
/// The status of the operation.
/// </summary>
public JobStatus Status { get; set; }
public static RestoreJobDto FromJob(IRestoreJob job)
{
return SimpleMapper.Map(job, new RestoreJobDto());
}
}
}

28
src/Squidex/Areas/Api/Controllers/Backups/Models/RestoreRequest.cs

@ -0,0 +1,28 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.ComponentModel.DataAnnotations;
namespace Squidex.Areas.Api.Controllers.Backups.Models
{
public sealed class RestoreRequest
{
/// <summary>
/// The name of the app.
/// </summary>
[Required]
[RegularExpression("^[a-z0-9]+(\\-[a-z0-9]+)*$")]
public string Name { get; set; }
/// <summary>
/// The url to the restore file.
/// </summary>
[Required]
public Uri Url { get; set; }
}
}

71
src/Squidex/Areas/Api/Controllers/Backups/RestoreController.cs

@ -0,0 +1,71 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using NSwag.Annotations;
using Orleans;
using Squidex.Areas.Api.Controllers.Backups.Models;
using Squidex.Domain.Apps.Entities.Backup;
using Squidex.Infrastructure.Commands;
using Squidex.Infrastructure.Security;
using Squidex.Pipeline;
namespace Squidex.Areas.Api.Controllers.Backups
{
/// <summary>
/// Restores backups.
/// </summary>
[ApiAuthorize]
[ApiExceptionFilter]
[ApiModelValidation(true)]
[MustBeAdministrator]
[SwaggerIgnore]
public class RestoreController : ApiController
{
private readonly IGrainFactory grainFactory;
public RestoreController(ICommandBus commandBus, IGrainFactory grainFactory)
: base(commandBus)
{
this.grainFactory = grainFactory;
}
[HttpGet]
[Route("apps/restore/")]
[ApiCosts(0)]
public async Task<IActionResult> GetJob()
{
var restoreGrain = grainFactory.GetGrain<IRestoreGrain>(User.OpenIdSubject());
var job = await restoreGrain.GetJobAsync();
if (job.Value == null)
{
return NotFound();
}
var jobs = await restoreGrain.GetJobAsync();
var response = RestoreJobDto.FromJob(job.Value);
return Ok(response);
}
[HttpPost]
[Route("apps/restore/")]
[ApiCosts(0)]
public async Task<IActionResult> PostRestore([FromBody] RestoreRequest request)
{
var restoreGrain = grainFactory.GetGrain<IRestoreGrain>(User.OpenIdSubject());
await restoreGrain.RestoreAsync(request.Url, request.Name);
return NoContent();
}
}
}

75
src/Squidex/Config/Domain/EntitiesServices.cs

@ -52,8 +52,7 @@ namespace Squidex.Config.Domain
c.GetRequiredService<IOptions<MyUrlsOptions>>(),
c.GetRequiredService<IAssetStore>(),
exposeSourceUrl))
.As<IGraphQLUrlGenerator>()
.As<IRuleUrlGenerator>();
.As<IGraphQLUrlGenerator>().As<IRuleUrlGenerator>();
services.AddSingletonAs<CachingGraphQLService>()
.As<IGraphQLService>();
@ -94,6 +93,35 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<ImageTagGenerator>()
.As<ITagGenerator<CreateAsset>>();
services.AddSingletonAs<JintScriptEngine>()
.As<IScriptEngine>();
services.AddCommandPipeline();
services.AddBackupHandlers();
services.AddSingleton<Func<IGrainCallContext, string>>(DomainObjectGrainFormatter.Format);
services.AddSingleton(c =>
{
var uiOptions = c.GetRequiredService<IOptions<MyUIOptions>>();
var result = new InitialPatterns();
foreach (var pattern in uiOptions.Value.RegexSuggestions)
{
if (!string.IsNullOrWhiteSpace(pattern.Key) &&
!string.IsNullOrWhiteSpace(pattern.Value))
{
result[Guid.NewGuid()] = new AppPattern(pattern.Key, pattern.Value);
}
}
return result;
});
}
private static void AddCommandPipeline(this IServiceCollection services)
{
services.AddSingletonAs<InMemoryCommandBus>()
.As<ICommandBus>();
@ -118,6 +146,9 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<AssetCommandMiddleware>()
.As<ICommandMiddleware>();
services.AddSingletonAs<AppsByNameIndexCommandMiddleware>()
.As<ICommandMiddleware>();
services.AddSingletonAs<GrainCommandMiddleware<AppCommand, IAppGrain>>()
.As<ICommandMiddleware>();
@ -130,9 +161,6 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<GrainCommandMiddleware<RuleCommand, IRuleGrain>>()
.As<ICommandMiddleware>();
services.AddSingletonAs<AppsByNameIndexCommandMiddleware>()
.As<ICommandMiddleware>();
services.AddSingletonAs<AppsByUserIndexCommandMiddleware>()
.As<ICommandMiddleware>();
@ -150,29 +178,24 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<CreateProfileCommandMiddleware>()
.As<ICommandMiddleware>();
}
services.AddSingletonAs<JintScriptEngine>()
.As<IScriptEngine>();
services.AddSingleton<Func<IGrainCallContext, string>>(DomainObjectGrainFormatter.Format);
services.AddSingleton(c =>
private static void AddBackupHandlers(this IServiceCollection services)
{
var uiOptions = c.GetRequiredService<IOptions<MyUIOptions>>();
services.AddTransientAs<BackupApps>()
.As<BackupHandler>();
var result = new InitialPatterns();
services.AddTransientAs<BackupAssets>()
.As<BackupHandler>();
foreach (var pattern in uiOptions.Value.RegexSuggestions)
{
if (!string.IsNullOrWhiteSpace(pattern.Key) &&
!string.IsNullOrWhiteSpace(pattern.Value))
{
result[Guid.NewGuid()] = new AppPattern(pattern.Key, pattern.Value);
}
}
services.AddTransientAs<BackupContents>()
.As<BackupHandler>();
return result;
});
services.AddTransientAs<BackupRules>()
.As<BackupHandler>();
services.AddTransientAs<BackupSchemas>()
.As<BackupHandler>();
}
public static void AddMyMigrationServices(this IServiceCollection services)
@ -180,6 +203,9 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<Migrator>()
.AsSelf();
services.AddTransientAs<Rebuilder>()
.AsSelf();
services.AddTransientAs<MigrationPath>()
.As<IMigrationPath>();
@ -209,9 +235,6 @@ namespace Squidex.Config.Domain
services.AddTransientAs<StopEventConsumers>()
.As<IMigration>();
services.AddTransientAs<Rebuilder>()
.AsSelf();
}
}
}

4
src/Squidex/Pipeline/CommandMiddlewares/EnrichWithActorCommandMiddleware.cs

@ -56,14 +56,14 @@ namespace Squidex.Pipeline.CommandMiddlewares
{
var subjectId = httpContextAccessor.HttpContext.User.OpenIdSubject();
return subjectId == null ? null : new RefToken("subject", subjectId);
return subjectId == null ? null : new RefToken(RefTokenType.Subject, subjectId);
}
private RefToken FindActorFromClient()
{
var clientId = httpContextAccessor.HttpContext.User.OpenIdClientId();
return clientId == null ? null : new RefToken("client", clientId);
return clientId == null ? null : new RefToken(RefTokenType.Client, clientId);
}
}
}

5
src/Squidex/app/features/administration/administration-area.component.html

@ -12,6 +12,11 @@
<i class="nav-icon icon-user-o"></i> <div class="nav-text">Users</div>
</a>
</li>
<li class="nav-item">
<a class="nav-link" routerLink="restore" routerLinkActive="active">
<i class="nav-icon icon-backup"></i> <div class="nav-text">Restore</div>
</a>
</li>
</ul>
</div>

1
src/Squidex/app/features/administration/declarations.ts

@ -11,6 +11,7 @@ export * from './guards/user-must-exist.guard';
export * from './guards/unset-user.guard';
export * from './pages/event-consumers/event-consumers-page.component';
export * from './pages/restore/restore-page.component';
export * from './pages/users/user-page.component';
export * from './pages/users/users-page.component';

6
src/Squidex/app/features/administration/module.ts

@ -18,6 +18,7 @@ import {
EventConsumersPageComponent,
EventConsumersService,
EventConsumersState,
RestorePageComponent,
UnsetUserGuard,
UserMustExistGuard,
UserPageComponent,
@ -38,6 +39,10 @@ const routes: Routes = [
path: 'event-consumers',
component: EventConsumersPageComponent
},
{
path: 'restore',
component: RestorePageComponent
},
{
path: 'users',
component: UsersPageComponent,
@ -69,6 +74,7 @@ const routes: Routes = [
declarations: [
AdministrationAreaComponent,
EventConsumersPageComponent,
RestorePageComponent,
UserPageComponent,
UsersPageComponent
],

68
src/Squidex/app/features/administration/pages/restore/restore-page.component.html

@ -0,0 +1,68 @@
<sqx-title message="Restore Backup"></sqx-title>
<sqx-panel theme="light" desiredWidth="70rem">
<ng-container title>
Restore Backup
</ng-container>
<ng-container content>
<ng-container *ngIf="restoreJob; let job">
<div class="card section">
<div class="card-header">
<div class="row no-gutters">
<div class="col col-auto pr-2">
<div *ngIf="job.status === 'Started'" class="restore-status restore-status-pending spin">
<i class="icon-hour-glass"></i>
</div>
<div *ngIf="job.status === 'Failed'" class="restore-status restore-status-failed">
<i class="icon-exclamation"></i>
</div>
<div *ngIf="job.status === 'Completed'" class="restore-status restore-status-success">
<i class="icon-checkmark"></i>
</div>
</div>
<div class="col">
<h3>Last Restore Operation</h3>
</div>
<div class="col text-right restore-url">
{{job.url}}
</div>
</div>
</div>
<div class="card-body">
<div *ngFor="let row of job.log">
{{row}}
</div>
</div>
<div class="card-footer text-muted">
<div class="row">
<div class="col">
Started: {{job.started | sqxISODate}}
</div>
<div class="col text-right" *ngIf="job.stopped">
Stopped: {{job.stopped | sqxISODate}}
</div>
</div>
</div>
</div>
</ng-container>
<div class="table-items-row">
<form [formGroup]="restoreForm.form" (submit)="restore()">
<div class="row no-gutters">
<div class="col">
<input class="form-control" name="url" formControlName="url" placeholder="Url to backup" />
</div>
<div class="col pl-1">
<input class="form-control" name="name" formControlName="name" placeholder="Optional app name" />
</div>
<div class="col col-auto pl-1">
<button type="submit" class="btn btn-success" [disabled]="restoreForm.hasNoUrl | async">Restore Backup</button>
</div>
</div>
</form>
</div>
</ng-container>
</sqx-panel>

68
src/Squidex/app/features/administration/pages/restore/restore-page.component.scss

@ -0,0 +1,68 @@
@import '_vars';
@import '_mixins';
$circle-size: 2rem;
h3 {
margin: 0;
}
.section {
margin-bottom: .8rem;
}
.container {
padding-top: 2rem;
}
.card {
&-header {
h3 {
line-height: $circle-size;
}
}
&-footer {
font-size: .9rem;
}
&-body {
font-family: monospace;
background: $color-border;
max-height: 400px;
min-height: 300px;
overflow-y: scroll;
}
}
.restore {
&-status {
& {
@include circle($circle-size);
line-height: $circle-size + .1rem;
text-align: center;
font-size: .6 * $circle-size;
font-weight: normal;
background: $color-border;
color: $color-dark-foreground;
vertical-align: middle;
}
&-pending {
color: inherit;
}
&-failed {
background: $color-theme-error;
}
&-success {
background: $color-theme-green;
}
}
&-url {
@include truncate;
line-height: 30px;
}
}

68
src/Squidex/app/features/administration/pages/restore/restore-page.component.ts

@ -0,0 +1,68 @@
/*
* Squidex Headless CMS
*
* @license
* Copyright (c) Squidex UG (haftungsbeschränkt). All rights reserved.
*/
import { Component, OnDestroy, OnInit } from '@angular/core';
import { FormBuilder } from '@angular/forms';
import { Subscription, timer } from 'rxjs';
import { switchMap } from 'rxjs/operators';
import {
AuthService,
BackupsService,
DialogService,
RestoreDto,
RestoreForm
} from '@app/shared';
@Component({
selector: 'sqx-restore-page',
styleUrls: ['./restore-page.component.scss'],
templateUrl: './restore-page.component.html'
})
export class RestorePageComponent implements OnDestroy, OnInit {
private timerSubscription: Subscription;
public restoreJob: RestoreDto | null;
public restoreForm = new RestoreForm(this.formBuilder);
constructor(
public readonly authState: AuthService,
private readonly backupsService: BackupsService,
private readonly dialogs: DialogService,
private readonly formBuilder: FormBuilder
) {
}
public ngOnDestroy() {
this.timerSubscription.unsubscribe();
}
public ngOnInit() {
this.timerSubscription =
timer(0, 2000).pipe(switchMap(() => this.backupsService.getRestore()))
.subscribe(dto => {
if (dto !== null) {
this.restoreJob = dto;
}
});
}
public restore() {
const value = this.restoreForm.submit();
if (value) {
this.restoreForm.submitCompleted({});
this.backupsService.postRestore(value)
.subscribe(() => {
this.dialogs.notifyInfo('Restore started, it can take several minutes to complete.');
}, error => {
this.dialogs.notifyError(error);
});
}
}
}

4
src/Squidex/app/features/administration/services/users.service.ts

@ -24,10 +24,6 @@ export class UsersDto extends Model {
) {
super();
}
public with(value: Partial<UsersDto>): UsersDto {
return this.clone(value);
}
}
export class UserDto extends Model {

1
src/Squidex/app/features/rules/pages/events/rule-events-page.component.scss

@ -18,6 +18,7 @@ h3 {
&-dump {
margin-top: 1rem;
font-family: monospace;
font-size: .8rem;
font-weight: normal;
height: 20rem;

8
src/Squidex/app/features/settings/pages/backups/backups-page.component.html

@ -33,15 +33,15 @@
</div>
<div class="table-items-row" *ngFor="let backup of backups; trackBy: trackByBackup">
<div class="row no-gutter">
<div class="row">
<div class="col col-auto">
<div *ngIf="!backup.stopped" class="backup-status backup-status-pending spin">
<div *ngIf="backup.status === 'Started'" class="backup-status backup-status-pending spin">
<i class="icon-hour-glass"></i>
</div>
<div *ngIf="backup.stopped && backup.isFailed" class="backup-status backup-status-failed">
<div *ngIf="backup.status === 'Failed'" class="backup-status backup-status-failed">
<i class="icon-exclamation"></i>
</div>
<div *ngIf="backup.stopped && !backup.isFailed" class="backup-status backup-status-success">
<div *ngIf="backup.status === 'Completed'" class="backup-status backup-status-success">
<i class="icon-checkmark"></i>
</div>
</div>

8
src/Squidex/app/features/settings/pages/backups/backups-page.component.scss

@ -1,14 +1,14 @@
@import '_vars';
@import '_mixins';
$cicle-size: 2.8rem;
$circle-size: 2.8rem;
.backup-status {
& {
@include circle($cicle-size);
line-height: $cicle-size + .1rem;
@include circle($circle-size);
line-height: $circle-size + .1rem;
text-align: center;
font-size: .4 * $cicle-size;
font-size: .4 * $circle-size;
font-weight: normal;
background: $color-border;
color: $color-dark-foreground;

2
src/Squidex/app/features/settings/pages/languages/language.component.html

@ -40,7 +40,7 @@
<div class="col col-9">
<div class="fallback-languages" [sqxSortModel]="fallbackLanguages.mutableValues" *ngIf="fallbackLanguages.length > 0">
<div class="fallback-language" *ngFor="let language of fallbackLanguages">
<div class="row no-gutter">
<div class="row">
<div class="col">
{{language.englishName}}
</div>

4
src/Squidex/app/shared/components/history-list.component.scss

@ -15,6 +15,10 @@
margin-top: .25rem;
}
.user-ref {
padding-right: .25rem;
}
.event {
& {
color: $color-history;

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save