Backup fixes.

src/Squidex.Domain.Apps.Entities/Backup/BackupReader.cs

@@ -8,13 +8,14 @@
 using System;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Threading.Tasks;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using Squidex.Domain.Apps.Entities.Backup.Helpers;
+using Squidex.Domain.Apps.Entities.Backup.Model;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.Json;
+using Squidex.Infrastructure.Json.Objects;
 using Squidex.Infrastructure.States;
 
 #pragma warning disable SA1401 // Fields must be private
@@ -29,36 +30,6 @@ namespace Squidex.Domain.Apps.Entities.Backup
         private int readEvents;
         private int readAttachments;
 
-        private sealed class ComaptibleStoredEvent
-        {
-            [JsonProperty]
-            public string StreamName;
-
-            [JsonProperty]
-            public string EventPosition;
-
-            [JsonProperty]
-            public long EventStreamNumber;
-
-            [JsonProperty]
-            public CompatibleEventData Data;
-        }
-
-        private sealed class CompatibleEventData
-        {
-            [JsonProperty]
-            public string Type;
-
-            [JsonProperty]
-            public JRaw Payload;
-
-            [JsonProperty]
-            public EnvelopeHeaders Headers;
-
-            [JsonProperty]
-            public EnvelopeHeaders Metadata;
-        }
-
         public int ReadEvents
         {
             get { return readEvents; }
@@ -151,20 +122,34 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
                 using (var stream = eventEntry.Open())
                 {
-                    var storedEvent = serializer.Deserialize<ComaptibleStoredEvent>(stream);
+                    var (streamName, data) = serializer.Deserialize<CompatibleStoredEvent>(stream).ToEvent();
-
-                    var src = storedEvent.Data;
 
-                    var data = new EventData(src.Type, src.Headers ?? src.Metadata, src.Payload.ToString());
+                    MapHeaders(data);
 
-                    var eventStream = streamNameResolver.WithNewId(storedEvent.StreamName, guidMapper.NewGuidOrNull);
+                    var eventStream = streamNameResolver.WithNewId(streamName, guidMapper.NewGuidOrNull);
                     var eventEnvelope = formatter.Parse(data, true, guidMapper.NewGuidOrValue);
 
-                    await handler((eventStream, eventEnvelope));
+                    await handler((streamName, eventEnvelope));
                 }
 
                 readEvents++;
             }
         }
+
+        private void MapHeaders(EventData data)
+        {
+            foreach (var kvp in data.Headers.ToList())
+            {
+                if (kvp.Value.Type == JsonValueType.String)
+                {
+                    var newGuid = guidMapper.NewGuidOrNull(kvp.Value.ToString());
+
+                    if (newGuid != null)
+                    {
+                        data.Headers.Add(kvp.Key, newGuid);
+                    }
+                }
+            }
+        }
     }
 }

src/Squidex.Domain.Apps.Entities/Backup/BackupVersion.cs

@@ -0,0 +1,15 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Domain.Apps.Entities.Backup
+{
+    public enum BackupVersion
+    {
+        V2,
+        V1
+    }
+}

src/Squidex.Domain.Apps.Entities/Backup/BackupWriter.cs

@@ -10,6 +10,7 @@ using System.IO;
 using System.IO.Compression;
 using System.Threading.Tasks;
 using Squidex.Domain.Apps.Entities.Backup.Helpers;
+using Squidex.Domain.Apps.Entities.Backup.Model;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.Json;
@@ -21,6 +22,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
     {
         private readonly ZipArchive archive;
         private readonly IJsonSerializer serializer;
+        private readonly Func<StoredEvent, CompatibleStoredEvent> converter;
         private int writtenEvents;
         private int writtenAttachments;
 
@@ -34,12 +36,17 @@ namespace Squidex.Domain.Apps.Entities.Backup
             get { return writtenAttachments; }
         }
 
-        public BackupWriter(IJsonSerializer serializer, Stream stream, bool keepOpen = false)
+        public BackupWriter(IJsonSerializer serializer, Stream stream, bool keepOpen = false, BackupVersion version = BackupVersion.V2)
         {
             Guard.NotNull(serializer, nameof(serializer));
 
             this.serializer = serializer;
 
+            converter =
+                version == BackupVersion.V1 ?
+                    new Func<StoredEvent, CompatibleStoredEvent>(CompatibleStoredEvent.V1) :
+                    new Func<StoredEvent, CompatibleStoredEvent>(CompatibleStoredEvent.V2);
+
             archive = new ZipArchive(stream, ZipArchiveMode.Create, keepOpen);
         }
 
@@ -90,7 +97,9 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
             using (var stream = eventEntry.Open())
             {
-                serializer.Serialize(storedEvent, stream);
+                var @event = converter(storedEvent);
+
+                serializer.Serialize(@event, stream);
             }
 
             writtenEvents++;

src/Squidex.Domain.Apps.Entities/Backup/GuidMapper.cs

@@ -19,7 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
         public Guid OldGuid(Guid newGuid)
         {
-            return newToOldGuid.GetOrDefault(newGuid);
+            return newToOldGuid.GetOrCreate(newGuid, x => x);
         }
 
         public string NewGuidOrNull(string value)

src/Squidex.Domain.Apps.Entities/Backup/Model/CompatibleStoredEvent.cs

@@ -0,0 +1,116 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Squidex.Infrastructure.EventSourcing;
+
+#pragma warning disable SA1401 // Fields must be private
+
+namespace Squidex.Domain.Apps.Entities.Backup.Model
+{
+    public sealed class CompatibleStoredEvent
+    {
+        [JsonProperty("n")]
+        public NewEvent NewEvent;
+
+        [JsonProperty]
+        public string StreamName;
+
+        [JsonProperty]
+        public string EventPosition;
+
+        [JsonProperty]
+        public long EventStreamNumber;
+
+        [JsonProperty]
+        public CompatibleEventData Data;
+
+        public static CompatibleStoredEvent V1(StoredEvent stored)
+        {
+            return new CompatibleStoredEvent
+            {
+                Data = CompatibleEventData.V1(stored.Data),
+                EventPosition = stored.EventPosition,
+                EventStreamNumber = stored.EventStreamNumber,
+                StreamName = stored.StreamName
+            };
+        }
+
+        public static CompatibleStoredEvent V2(StoredEvent stored)
+        {
+            return new CompatibleStoredEvent { NewEvent = NewEvent.V2(stored) };
+        }
+
+        public (string Stream, EventData Data) ToEvent()
+        {
+            if (NewEvent != null)
+            {
+                return NewEvent.ToEvent();
+            }
+            else
+            {
+                return (StreamName, Data.ToData());
+            }
+        }
+    }
+
+    public sealed class CompatibleEventData
+    {
+        [JsonProperty]
+        public string Type;
+
+        [JsonProperty]
+        public JRaw Payload;
+
+        [JsonProperty]
+        public EnvelopeHeaders Metadata;
+
+        public static CompatibleEventData V1(EventData data)
+        {
+            var payload = new JRaw(data.Payload);
+
+            return new CompatibleEventData { Type = data.Type, Payload = payload, Metadata = data.Headers };
+        }
+
+        public EventData ToData()
+        {
+            return new EventData(Type, Metadata, Payload.ToString());
+        }
+    }
+
+    public sealed class NewEvent
+    {
+        [JsonProperty("t")]
+        public string EventType;
+
+        [JsonProperty("s")]
+        public string StreamName;
+
+        [JsonProperty("p")]
+        public string EventPayload;
+
+        [JsonProperty("h")]
+        public EnvelopeHeaders EventHeaders;
+
+        public static NewEvent V2(StoredEvent stored)
+        {
+            return new NewEvent
+            {
+                EventType = stored.Data.Type,
+                EventHeaders = stored.Data.Headers,
+                EventPayload = stored.Data.Payload,
+                StreamName = stored.StreamName
+            };
+        }
+
+        public (string Stream, EventData Data) ToEvent()
+        {
+            return (StreamName, new EventData(EventType, EventHeaders, EventPayload));
+        }
+    }
+}

src/Squidex.Infrastructure/EventSourcing/DefaultEventDataFormatter.cs

@@ -10,7 +10,7 @@ using Squidex.Infrastructure.Json;
 
 namespace Squidex.Infrastructure.EventSourcing
 {
-    public class DefaultEventDataFormatter : IEventDataFormatter
+    public sealed class DefaultEventDataFormatter : IEventDataFormatter
     {
         private readonly IJsonSerializer serializer;
         private readonly TypeNameRegistry typeNameRegistry;
@@ -28,14 +28,14 @@ namespace Squidex.Infrastructure.EventSourcing
         public Envelope<IEvent> Parse(EventData eventData, bool migrate = true, Func<string, string> stringConverter = null)
         {
             var payloadType = typeNameRegistry.GetType(eventData.Type);
-            var payload = serializer.Deserialize<IEvent>(eventData.Payload, payloadType, stringConverter);
+            var payloadObj = serializer.Deserialize<IEvent>(eventData.Payload, payloadType, stringConverter);
 
-            if (migrate && payload is IMigratedEvent migratedEvent)
+            if (migrate && payloadObj is IMigratedEvent migratedEvent)
             {
-                payload = migratedEvent.Migrate();
+                payloadObj = migratedEvent.Migrate();
             }
 
-            var envelope = new Envelope<IEvent>(payload, eventData.Headers);
+            var envelope = new Envelope<IEvent>(payloadObj, eventData.Headers);
 
             return envelope;
         }
@@ -50,11 +50,11 @@ namespace Squidex.Infrastructure.EventSourcing
             }
 
             var payloadType = typeNameRegistry.GetName(eventPayload.GetType());
-            var payload = serializer.Serialize(envelope.Payload);
+            var payloadJson = serializer.Serialize(envelope.Payload);
 
             envelope.SetCommitId(commitId);
 
-            return new EventData(payloadType, envelope.Headers, payload);
+            return new EventData(payloadType, envelope.Headers, payloadJson);
         }
     }
 }

src/Squidex.Infrastructure/Queries/FilterValue.cs

@@ -8,6 +8,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using NodaTime;
 
@@ -126,26 +127,23 @@ namespace Squidex.Infrastructure.Queries
             {
                 return $"[{string.Join(", ", list.OfType<object>().Select(ToString).ToArray())}]";
             }
-            else
+
-            {
+            return ToString(Value);
-                return ToString(Value);
-            }
         }
 
         private string ToString(object value)
         {
-            if (ValueType == FilterValueType.String)
+            if (value == null)
-            {
-                return $"'{value.ToString().Replace("'", "\\'")}'";
-            }
-            else if (value == null)
             {
                 return "null";
             }
-            else
+
+            if (value is string s)
             {
-                return value.ToString();
+                return $"'{s.Replace("'", "\\'")}'";
             }
+
+            return string.Format(CultureInfo.InvariantCulture, "{0}", value);
         }
     }
 }

src/Squidex/appsettings.json

@@ -253,7 +253,7 @@
 
   "identity": {
     /*
-     * Enable password auth.  Set this to false if you want to disable local login, leaving only 3rd party login options.
+     * Enable password auth. Set this to false if you want to disable local login, leaving only 3rd party login options.
      */
     "allowPasswordAuth": true,
     /*

tests/Squidex.Domain.Apps.Core.Tests/Operations/ConvertContent/ContentConversionTests.cs

@@ -174,7 +174,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
                 new NamedContentData()
                     .AddField("field1",
                         new ContentFieldData()
-                            .AddValue("en", new JsonArray("hello", "loved")))
+                            .AddValue("en", JsonValue.Array("hello", "loved")))
                     .AddField("field2",
                         new ContentFieldData()
                             .AddValue("iv", "world"));
@@ -191,7 +191,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
                 new NamedContentData()
                     .AddField("field1",
                         new ContentFieldData()
-                            .AddValue("en", new JsonArray(JsonValue.Object().Add("p1", "hello"))))
+                            .AddValue("en", JsonValue.Array(JsonValue.Object().Add("p1", "hello"))))
                     .AddField("field2",
                         new ContentFieldData()
                             .AddValue("iv", "world"));

tests/Squidex.Domain.Apps.Core.Tests/Operations/ConvertContent/FieldConvertersTests.cs

@@ -73,7 +73,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var input =
                 new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("field1", 100)
                                 .Add("field2", 200)
@@ -84,7 +84,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var expected =
                new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("1", 100)));
 
@@ -97,7 +97,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var input =
                 new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("field1", 100)
                                 .Add("field2", 200)
@@ -108,7 +108,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var expected =
                 new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("field1", 100)));
 
@@ -121,7 +121,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var input =
                new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("1", 100)
                                 .Add("2", 200)
@@ -132,7 +132,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var expected =
                new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("1", 100)));
 
@@ -145,7 +145,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var input =
                new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("1", 100)
                                 .Add("2", 200)
@@ -156,7 +156,7 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
             var expected =
                new ContentFieldData()
                     .AddValue("iv",
-                        new JsonArray(
+                        JsonValue.Array(
                             JsonValue.Object()
                                 .Add("field1", 100)));
 
@@ -422,11 +422,11 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
         {
             var source =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("1", "2"));
+                    .AddValue("iv", JsonValue.Array("1", "2"));
 
             var expected =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("url/to/1", "url/to/2"));
+                    .AddValue("iv", JsonValue.Array("url/to/1", "url/to/2"));
 
             var rtesult = FieldConverters.ResolveAssetUrls(new HashSet<string>(new[] { "1" }), assetUrlGenerator)(source, assetsField);
 
@@ -438,11 +438,11 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
         {
             var source =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("1", "2"));
+                    .AddValue("iv", JsonValue.Array("1", "2"));
 
             var expected =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("url/to/1", "url/to/2"));
+                    .AddValue("iv", JsonValue.Array("url/to/1", "url/to/2"));
 
             var rtesult = FieldConverters.ResolveAssetUrls(new HashSet<string>(new[] { "*" }), assetUrlGenerator)(source, assetsField);
 
@@ -454,11 +454,11 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
         {
             var source =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("1", "2"));
+                    .AddValue("iv", JsonValue.Array("1", "2"));
 
             var expected =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("1", "2"));
+                    .AddValue("iv", JsonValue.Array("1", "2"));
 
             var rtesult = FieldConverters.ResolveAssetUrls(new HashSet<string>(new[] { "2" }), assetUrlGenerator)(source, assetsField);
 
@@ -470,11 +470,11 @@ namespace Squidex.Domain.Apps.Core.Operations.ConvertContent
         {
             var source =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("1", "2"));
+                    .AddValue("iv", JsonValue.Array("1", "2"));
 
             var expected =
                 new ContentFieldData()
-                    .AddValue("iv", new JsonArray("1", "2"));
+                    .AddValue("iv", JsonValue.Array("1", "2"));
 
             var rtesult = FieldConverters.ResolveAssetUrls(null, assetUrlGenerator)(source, assetsField);
 

tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleServiceTests.cs

@@ -171,7 +171,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules
         {
             var @event = new ContentCreated { SchemaId = NamedId.Of(Guid.NewGuid(), "my-schema"), AppId = NamedId.Of(Guid.NewGuid(), "my-event") };
 
-            var now = SystemClock.Instance.GetCurrentInstant();
+            var now = Instant.FromUnixTimeSeconds(SystemClock.Instance.GetCurrentInstant().ToUnixTimeSeconds());
 
             var ruleConfig = ValidRule();
             var ruleEnvelope = Envelope.Create(@event);

tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/ContentDataObjectTests.cs

@@ -155,13 +155,13 @@ namespace Squidex.Domain.Apps.Core.Operations.Scripting
                 new NamedContentData()
                     .AddField("number",
                         new ContentFieldData()
-                            .AddValue("iv", new JsonArray(1.0, 2.0)));
+                            .AddValue("iv", JsonValue.Array(1.0, 2.0)));
 
             var expected =
                 new NamedContentData()
                     .AddField("number",
                         new ContentFieldData()
-                            .AddValue("iv", new JsonArray(1.0, 4.0, 5.0)));
+                            .AddValue("iv", JsonValue.Array(1.0, 4.0, 5.0)));
 
             var result = ExecuteScript(original, @"data.number.iv = [data.number.iv[0], data.number.iv[1] + 2, 5]");
 
@@ -278,7 +278,7 @@ namespace Squidex.Domain.Apps.Core.Operations.Scripting
                 new NamedContentData()
                     .AddField("obj",
                         new ContentFieldData()
-                            .AddValue("iv", new JsonArray()));
+                            .AddValue("iv", JsonValue.Array()));
 
             ExecuteScript(original, "data.obj.iv[0] = 1");
         }

tests/Squidex.Domain.Apps.Entities.Tests/Backup/BackupReaderWriterTests.cs

@@ -47,8 +47,10 @@ namespace Squidex.Domain.Apps.Entities.Backup
                 .ReturnsLazily(new Func<string, Func<string, string>, string>((stream, idGenerator) => stream + "^2"));
         }
 
-        [Fact]
+        [Theory]
-        public async Task Should_write_and_read_events()
+        [InlineData(BackupVersion.V1)]
+        [InlineData(BackupVersion.V2)]
+        public async Task Should_write_and_read_events_to_backup(BackupVersion version)
         {
             var stream = new MemoryStream();
 
@@ -83,13 +85,13 @@ namespace Squidex.Domain.Apps.Entities.Backup
                 var envelope = Envelope.Create<IEvent>(@event);
 
                 envelope.Headers.Add(RandomGuid().ToString(), i);
-                envelope.Headers.Add("Id", RandomGuid());
+                envelope.Headers.Add("Id", RandomGuid().ToString());
                 envelope.Headers.Add("Index", i);
 
                 sourceEvents.Add(($"My-{RandomGuid()}", envelope));
             }
 
-            using (var writer = new BackupWriter(serializer, stream, true))
+            using (var writer = new BackupWriter(serializer, stream, true, version))
             {
                 foreach (var @event in sourceEvents)
                 {
@@ -149,13 +151,18 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
                 for (var i = 0; i < targetEvents.Count; i++)
                 {
-                    var lhs = targetEvents[i].Event.To<MyEvent>();
+                    var tgt = targetEvents[i].Event.To<MyEvent>();
-                    var rhs = sourceEvents[i].Event.To<MyEvent>();
+                    var src = sourceEvents[i].Event.To<MyEvent>();
 
-                    Assert.Equal(rhs.Payload.GuidRaw, reader.OldGuid(lhs.Payload.GuidRaw));
+                    Assert.Equal(src.Payload.GuidRaw, reader.OldGuid(tgt.Payload.GuidRaw));
-                    Assert.Equal(rhs.Payload.GuidNamed.Id, reader.OldGuid(lhs.Payload.GuidNamed.Id));
+                    Assert.Equal(src.Payload.GuidNamed.Id, reader.OldGuid(tgt.Payload.GuidNamed.Id));
 
-                    Assert.Equal(rhs.Headers.GetGuid("Id"), reader.OldGuid(lhs.Headers.GetGuid("Id")));
+                    Assert.NotEqual(src.Payload.GuidRaw, tgt.Payload.GuidRaw);
+                    Assert.NotEqual(src.Payload.GuidNamed.Id, tgt.Payload.GuidNamed.Id);
+
+                    Assert.Equal(src.Headers.GetGuid("Id"), reader.OldGuid(tgt.Headers.GetGuid("Id")));
+
+                    Assert.NotEqual(src.Headers.GetGuid("Id"), tgt.Headers.GetGuid("Id"));
                 }
             }
         }

tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj

@@ -5,11 +5,6 @@
     <RuntimeFrameworkVersion>2.1.1</RuntimeFrameworkVersion>
     <RootNamespace>Squidex.Domain.Apps.Entities</RootNamespace>
   </PropertyGroup>
-  <ItemGroup>
-    <Compile Remove="MongoDb\**" />
-    <EmbeddedResource Remove="MongoDb\**" />
-    <None Remove="MongoDb\**" />
-  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\Squidex.Domain.Apps.Core.Model\Squidex.Domain.Apps.Core.Model.csproj" />
     <ProjectReference Include="..\..\src\Squidex.Domain.Apps.Core.Operations\Squidex.Domain.Apps.Core.Operations.csproj" />