Backup check.

backend/src/Squidex.Domain.Apps.Entities/Apps/BackupApps.cs

@@ -109,7 +109,7 @@ namespace Squidex.Domain.Apps.Entities.Apps
 
         public async Task RestoreAsync(RestoreContext context)
         {
-            var json = await context.Reader.ReadJsonAttachmentAsync<JsonObject>(SettingsFile);
+            var json = await context.Reader.ReadJsonAsync<JsonObject>(SettingsFile);
 
             await appUISettings.SetAsync(context.AppId, null, json);
         }

backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs

@@ -110,7 +110,7 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
         private async Task RestoreTagsAsync(RestoreContext context)
         {
-            var tags = await context.Reader.ReadJsonAttachmentAsync<TagsExport>(TagsFile);
+            var tags = await context.Reader.ReadJsonAsync<TagsExport>(TagsFile);
 
             await tagService.RebuildTagsAsync(context.AppId, TagGroups.Assets, tags);
         }

backend/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs

@@ -136,12 +136,12 @@ namespace Squidex.Domain.Apps.Entities.Backup
                 {
                     using (var writer = await backupArchiveLocation.OpenWriterAsync(stream))
                     {
+                        await writer.WriteVersionAsync();
+
                         var userMapping = new UserMapping(actor);
 
                         var context = new BackupContext(Key, userMapping, writer);
 
-                        var filter = $"^[^\\-]*-{Regex.Escape(Key)}";
-
                         await eventStore.QueryAsync(async storedEvent =>
                         {
                             var @event = eventDataFormatter.Parse(storedEvent.Data);
@@ -162,7 +162,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
                             job.HandledAssets = writer.WrittenAttachments;
 
                             lastTimestamp = await WritePeriodically(lastTimestamp);
-                        }, filter, null, ct);
+                        }, GetFilter(), null, ct);
 
                         foreach (var handler in handlers)
                         {
@@ -215,6 +215,11 @@ namespace Squidex.Domain.Apps.Entities.Backup
             }
         }
 
+        private string GetFilter()
+        {
+            return $"^[^\\-]*-{Regex.Escape(Key)}";
+        }
+
         private async Task<Instant> WritePeriodically(Instant lastTimestamp)
         {
             var now = clock.GetCurrentInstant();

backend/src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs

@@ -54,27 +54,16 @@ namespace Squidex.Domain.Apps.Entities.Backup
             }
         }
 
-        public Task<T> ReadJsonAttachmentAsync<T>(string name)
+        public Task<T> ReadJsonAsync<T>(string name)
         {
             Guard.NotNullOrEmpty(name, nameof(name));
 
-            var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name));
+            var entry = GetEntry(name);
 
-            if (attachmentEntry == null)
+            using (var stream = entry.Open())
             {
-                throw new FileNotFoundException("Cannot find attachment.", name);
+                return Task.FromResult(serializer.Deserialize<T>(stream, null));
             }
-
-            T result;
-
-            using (var stream = attachmentEntry.Open())
-            {
-                result = serializer.Deserialize<T>(stream, null);
-            }
-
-            readAttachments++;
-
-            return Task.FromResult(result);
         }
 
         public async Task ReadBlobAsync(string name, Func<Stream, Task> handler)
@@ -82,19 +71,26 @@ namespace Squidex.Domain.Apps.Entities.Backup
             Guard.NotNullOrEmpty(name, nameof(name));
             Guard.NotNull(handler, nameof(handler));
 
-            var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name));
+            var entry = GetEntry(name);
 
-            if (attachmentEntry == null)
+            using (var stream = entry.Open())
             {
-                throw new FileNotFoundException("Cannot find attachment.", name);
+                await handler(stream);
             }
+        }
 
-            using (var stream = attachmentEntry.Open())
+        private ZipArchiveEntry GetEntry(string name)
+        {
+            var attachmentEntry = archive.GetEntry(ArchiveHelper.GetAttachmentPath(name));
+
+            if (attachmentEntry == null)
             {
-                await handler(stream);
+                throw new FileNotFoundException("Cannot find attachment.", name);
             }
 
             readAttachments++;
+
+            return attachmentEntry;
         }
 
         public async Task ReadEventsAsync(IStreamNameResolver streamNameResolver, IEventDataFormatter formatter, Func<(string Stream, Envelope<IEvent> Event), Task> handler)

backend/src/Squidex.Domain.Apps.Entities/Backup/CompatibilityExtensions.cs

@@ -0,0 +1,55 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Threading.Tasks;
+
+namespace Squidex.Domain.Apps.Entities.Backup
+{
+    public static class CompatibilityExtensions
+    {
+        private const string VersionFile = "Version.json";
+        private static readonly FileVersion None = new FileVersion();
+        private static readonly FileVersion Expected = new FileVersion { Major = 5 };
+
+        public sealed class FileVersion
+        {
+            public int Major { get; set; }
+
+            public bool Equals(FileVersion other)
+            {
+                return Major == other.Major;
+            }
+        }
+
+        public static Task WriteVersionAsync(this IBackupWriter writer)
+        {
+            return writer.WriteJsonAsync(VersionFile, Expected);
+        }
+
+        public static async Task CheckCompatibilityAsync(this IBackupReader reader)
+        {
+            var current = await reader.ReadVersionAsync();
+
+            if (!Expected.Equals(current))
+            {
+                throw new BackupRestoreException("Backup file is not compatible with this version.");
+            }
+        }
+
+        private static async Task<FileVersion> ReadVersionAsync(this IBackupReader reader)
+        {
+            try
+            {
+                return await reader.ReadJsonAsync<FileVersion>(VersionFile);
+            }
+            catch
+            {
+                return None;
+            }
+        }
+    }
+}

backend/src/Squidex.Domain.Apps.Entities/Backup/IBackupReader.cs

@@ -23,6 +23,6 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
         Task ReadEventsAsync(IStreamNameResolver streamNameResolver, IEventDataFormatter formatter, Func<(string Stream, Envelope<IEvent> Event), Task> handler);
 
-        Task<T> ReadJsonAttachmentAsync<T>(string name);
+        Task<T> ReadJsonAsync<T>(string name);
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs

@@ -161,6 +161,8 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
                     using (var reader = await DownloadAsync())
                     {
+                        await reader.CheckCompatibilityAsync();
+
                         using (Profiler.Trace("ReadEvents"))
                         {
                             await ReadEventsAsync(reader, handlers);

backend/src/Squidex.Domain.Apps.Entities/Backup/UserMapping.cs

@@ -70,7 +70,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
             Guard.NotNull(reader, nameof(reader));
             Guard.NotNull(userResolver, nameof(userResolver));
 
-            var json = await reader.ReadJsonAttachmentAsync<Dictionary<string, string>>(UsersFile);
+            var json = await reader.ReadJsonAsync<Dictionary<string, string>>(UsersFile);
 
             foreach (var (userId, email) in json)
             {

backend/src/Squidex.Domain.Apps.Entities/Contents/BackupContents.cs

@@ -224,7 +224,7 @@ namespace Squidex.Domain.Apps.Entities.Contents
         {
             try
             {
-                return await reader.ReadJsonAttachmentAsync<Urls>(UrlsFile);
+                return await reader.ReadJsonAsync<Urls>(UrlsFile);
             }
             catch
             {

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs

@@ -69,10 +69,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
         {
             var data = new
             {
-                appId = upsert.AppId.Id,
+                appId = upsert.AppId.Id.ToString(),
                 appName = upsert.AppId.Name,
-                contentId = upsert.ContentId,
-                schemaId = upsert.SchemaId.Id,
+                contentId = upsert.ContentId.ToString(),
+                schemaId = upsert.SchemaId.Id.ToString(),
                 schemaName = upsert.SchemaId.Name,
                 serveAll = upsert.ServeAll,
                 servePublished = upsert.ServePublished,

backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/BackupAppsTests.cs

@@ -155,7 +155,7 @@ namespace Squidex.Domain.Apps.Entities.Apps
 
             var context = CreateRestoreContext();
 
-            A.CallTo(() => context.Reader.ReadJsonAttachmentAsync<JsonObject>(A<string>._))
+            A.CallTo(() => context.Reader.ReadJsonAsync<JsonObject>(A<string>._))
                 .Returns(settings);
 
             await sut.RestoreAsync(context);

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/BackupAssetsTests.cs

@@ -67,7 +67,7 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
             var context = CreateRestoreContext();
 
-            A.CallTo(() => context.Reader.ReadJsonAttachmentAsync<TagsExport>(A<string>._))
+            A.CallTo(() => context.Reader.ReadJsonAsync<TagsExport>(A<string>._))
                 .Returns(tags);
 
             await sut.RestoreAsync(context);

backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupCompatibilityTests.cs

@@ -0,0 +1,62 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.IO;
+using System.Threading.Tasks;
+using FakeItEasy;
+using Xunit;
+
+namespace Squidex.Domain.Apps.Entities.Backup
+{
+    public class BackupCompatibilityTests
+    {
+        [Fact]
+        public async Task Should_writer_version()
+        {
+            var writer = A.Fake<IBackupWriter>();
+
+            await writer.WriteVersionAsync();
+
+            A.CallTo(() => writer.WriteJsonAsync(A<string>._,
+                    A<CompatibilityExtensions.FileVersion>.That.Matches(x => x.Major == 5)))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_not_throw_exception_if_backup_has_correct_version()
+        {
+            var reader = A.Fake<IBackupReader>();
+
+            A.CallTo(() => reader.ReadJsonAsync<CompatibilityExtensions.FileVersion>(A<string>._))
+                .Returns(new CompatibilityExtensions.FileVersion { Major = 5 });
+
+            await reader.CheckCompatibilityAsync();
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_backup_has_wrong_version()
+        {
+            var reader = A.Fake<IBackupReader>();
+
+            A.CallTo(() => reader.ReadJsonAsync<CompatibilityExtensions.FileVersion>(A<string>._))
+                .Returns(new CompatibilityExtensions.FileVersion { Major = 3 });
+
+            await Assert.ThrowsAsync<BackupRestoreException>(() => reader.CheckCompatibilityAsync());
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_backup_has_no_version()
+        {
+            var reader = A.Fake<IBackupReader>();
+
+            A.CallTo(() => reader.ReadJsonAsync<CompatibilityExtensions.FileVersion>(A<string>._))
+                .Throws(new FileNotFoundException());
+
+            await Assert.ThrowsAsync<BackupRestoreException>(() => reader.CheckCompatibilityAsync());
+        }
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs

@@ -8,7 +8,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Threading.Tasks;
 using FakeItEasy;
 using Squidex.Domain.Apps.Core.TestHelpers;
@@ -31,13 +30,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
         [TypeName(nameof(MyEvent))]
         public sealed class MyEvent : IEvent
         {
-            public DomainId DomainIdRaw { get; set; }
-
-            public DomainId DomainIdEmpty { get; set; }
-
-            public NamedId<DomainId> DomainIdNamed { get; set; }
-
-            public Dictionary<DomainId, string> Values { get; set; }
+            public Guid Id { get; set; } = Guid.NewGuid();
         }
 
         public BackupReaderWriterTests()
@@ -47,14 +40,72 @@ namespace Squidex.Domain.Apps.Entities.Backup
             formatter = new DefaultEventDataFormatter(typeNameRegistry, serializer);
         }
 
+        [Fact]
+        public async Task Should_read_and_write_json_async()
+        {
+            var file = "File.json";
+
+            var value = Guid.NewGuid();
+
+            await TestReaderWriterAsync(BackupVersion.V1, async writer =>
+            {
+                await WriteJsonGuidAsync(writer, file, value);
+            }, async reader =>
+            {
+                var read = await ReadJsonGuidAsync(reader, file);
+
+                Assert.Equal(value, read);
+            });
+        }
+
+        [Fact]
+        public async Task Should_read_and_write_blob_async()
+        {
+            var file = "File.json";
+
+            var value = Guid.NewGuid();
+
+            await TestReaderWriterAsync(BackupVersion.V1, async writer =>
+            {
+                await WriteGuidAsync(writer, file, value);
+            }, async reader =>
+            {
+                var read = await ReadGuidAsync(reader, file);
+
+                Assert.Equal(value, read);
+            });
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_json_not_found()
+        {
+            await TestReaderWriterAsync(BackupVersion.V1, writer =>
+            {
+                return Task.CompletedTask;
+            }, async reader =>
+            {
+                await Assert.ThrowsAsync<FileNotFoundException>(() => reader.ReadJsonAsync<int>("404"));
+            });
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_blob_not_found()
+        {
+            await TestReaderWriterAsync(BackupVersion.V1, writer =>
+            {
+                return Task.CompletedTask;
+            }, async reader =>
+            {
+                await Assert.ThrowsAsync<FileNotFoundException>(() => reader.ReadBlobAsync("404", s => Task.CompletedTask));
+            });
+        }
+
         [Theory]
         [InlineData(BackupVersion.V1)]
         [InlineData(BackupVersion.V2)]
         public async Task Should_write_and_read_events_to_backup(BackupVersion version)
         {
-            var stream = new MemoryStream();
-
-            var random = new Random();
+            var randomGenerator = new Random();
             var randomDomainIds = new List<DomainId>();
 
             for (var i = 0; i < 100; i++)
@@ -64,85 +115,64 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
             DomainId RandomDomainId()
             {
-                return randomDomainIds[random.Next(randomDomainIds.Count)];
+                return randomDomainIds[randomGenerator.Next(randomDomainIds.Count)];
             }
 
-            var sourceEvents = new List<(string Stream, Envelope<IEvent> Event)>();
+            var sourceEvents = new List<(string Stream, Envelope<MyEvent> Event)>();
 
             for (var i = 0; i < 200; i++)
             {
-                var @event = new MyEvent
-                {
-                    DomainIdNamed = NamedId.Of(RandomDomainId(), $"name{i}"),
-                    DomainIdRaw = RandomDomainId(),
-                    Values = new Dictionary<DomainId, string>
-                    {
-                        [RandomDomainId()] = "Key"
-                    }
-                };
+                var @event = new MyEvent();
 
-                var envelope = Envelope.Create<IEvent>(@event);
+                var envelope = Envelope.Create(@event);
 
-                envelope.Headers.Add(RandomDomainId().ToString(), i);
-                envelope.Headers.Add("Id", RandomDomainId().ToString());
+                envelope.Headers.Add("Id", @event.Id.ToString());
                 envelope.Headers.Add("Index", i);
 
                 sourceEvents.Add(($"My-{RandomDomainId()}", envelope));
             }
 
-            using (var writer = new BackupWriter(serializer, stream, true, version))
+            await TestReaderWriterAsync(version, async writer =>
             {
-                foreach (var (_, envelope) in sourceEvents)
+                foreach (var (stream, envelope) in sourceEvents)
                 {
                     var eventData = formatter.ToEventData(envelope, Guid.NewGuid(), true);
-                    var eventStored = new StoredEvent("S", "1", 2, eventData);
+                    var eventStored = new StoredEvent(stream, "1", 2, eventData);
 
                     var index = int.Parse(envelope.Headers["Index"].ToString());
 
                     if (index % 17 == 0)
                     {
-                        await writer.WriteBlobAsync(index.ToString(), innerStream =>
-                        {
-                            innerStream.WriteByte((byte)index);
-
-                            return Task.CompletedTask;
-                        });
+                        await WriteGuidAsync(writer, index.ToString(), envelope.Payload.Id);
                     }
                     else if (index % 37 == 0)
                     {
-                        await writer.WriteJsonAsync(index.ToString(), $"JSON_{index}");
+                        await WriteJsonGuidAsync(writer, index.ToString(), envelope.Payload.Id);
                     }
 
                     writer.WriteEvent(eventStored);
                 }
-            }
-
-            stream.Position = 0;
-
-            var targetEvents = new List<(string Stream, Envelope<IEvent> Event)>();
-
-            using (var reader = new BackupReader(serializer, stream))
+            }, async reader =>
             {
+                var targetEvents = new List<(string Stream, Envelope<IEvent> Event)>();
+
                 await reader.ReadEventsAsync(streamNameResolver, formatter, async @event =>
                 {
                     var index = int.Parse(@event.Event.Headers["Index"].ToString());
 
+                    var id = Guid.Parse(@event.Event.Headers["Id"].ToString());
+
                     if (index % 17 == 0)
                     {
-                        await reader.ReadBlobAsync(index.ToString(), innerStream =>
-                        {
-                            var byteRead = innerStream.ReadByte();
-
-                            Assert.Equal((byte)index, byteRead);
+                        var guid = await ReadGuidAsync(reader, index.ToString());
 
-                            return Task.CompletedTask;
-                        });
+                        Assert.Equal(id, guid);
                     }
                     else if (index % 37 == 0)
                     {
-                        var json = await reader.ReadJsonAttachmentAsync<string>(index.ToString());
+                        var guid = await ReadJsonGuidAsync(reader, index.ToString());
 
-                        Assert.Equal($"JSON_{index}", json);
+                        Assert.Equal(id, guid);
                     }
 
                     targetEvents.Add(@event);
@@ -150,15 +180,66 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
                 for (var i = 0; i < targetEvents.Count; i++)
                 {
-                    var target = targetEvents[i].Event.To<MyEvent>();
+                    var targetEvent = targetEvents[i].Event.To<MyEvent>();
+                    var targetStream = targetEvents[i].Stream;
+
+                    var sourceEvent = sourceEvents[i].Event.To<MyEvent>();
+                    var sourceStream = sourceEvents[i].Stream;
+
+                    Assert.Equal(sourceEvent.Payload.Id, targetEvent.Payload.Id);
+                    Assert.Equal(sourceStream, targetStream);
+                }
+            });
+        }
+
+        private static Task<Guid> ReadJsonGuidAsync(IBackupReader reader, string file)
+        {
+            return reader.ReadJsonAsync<Guid>(file);
+        }
+
+        private static Task WriteJsonGuidAsync(IBackupWriter writer, string file, Guid value)
+        {
+            return writer.WriteJsonAsync(file, value);
+        }
 
-                    var source = sourceEvents[i].Event.To<MyEvent>();
+        private static Task WriteGuidAsync(IBackupWriter writer, string file, Guid value)
+        {
+            return writer.WriteBlobAsync(file, async stream =>
+            {
+                await stream.WriteAsync(value.ToByteArray());
+            });
+        }
 
-                    Assert.Equal(source.Payload.Values.First().Key, target.Payload.Values.First().Key);
-                    Assert.Equal(source.Payload.DomainIdRaw, target.Payload.DomainIdRaw);
-                    Assert.Equal(source.Payload.DomainIdNamed.Id, target.Payload.DomainIdNamed.Id);
+        private static async Task<Guid> ReadGuidAsync(IBackupReader reader, string file)
+        {
+            var read = Guid.Empty;
 
-                    Assert.Equal(DomainId.Empty, target.Payload.DomainIdEmpty);
+            await reader.ReadBlobAsync(file, async stream =>
+            {
+                var buffer = new byte[16];
+
+                await stream.ReadAsync(buffer);
+
+                read = new Guid(buffer);
+            });
+
+            return read;
+        }
+
+        private async Task TestReaderWriterAsync(BackupVersion version, Func<IBackupWriter, Task> write, Func<IBackupReader, Task> read)
+        {
+            using (var stream = new MemoryStream())
+            {
+                using (var writer = new BackupWriter(serializer, stream, true, version))
+                {
+                    await write(writer);
+                }
+
+                stream.Position = 0;
+
+                using (var reader = new BackupReader(serializer, stream))
+                {
+                    await read(reader);
                 }
             }
         }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Backup/UserMappingTests.cs

@@ -123,7 +123,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
             var reader = A.Fake<IBackupReader>();
 
-            A.CallTo(() => reader.ReadJsonAttachmentAsync<Dictionary<string, string>>(A<string>._))
+            A.CallTo(() => reader.ReadJsonAsync<Dictionary<string, string>>(A<string>._))
                 .Returns(storedUsers);
 
             return reader;

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/BackupContentsTests.cs

@@ -91,7 +91,7 @@ namespace Squidex.Domain.Apps.Entities.Contents
             A.CallTo(() => urlGenerator.AssetContentBase(appId.Name))
                 .Returns(newAssetsUrlApp);
 
-            A.CallTo(() => reader.ReadJsonAttachmentAsync<BackupContents.Urls>(A<string>._))
+            A.CallTo(() => reader.ReadJsonAsync<BackupContents.Urls>(A<string>._))
                 .Returns(new BackupContents.Urls
                 {
                     Assets = oldAssetsUrl,