Cleanup

src/Squidex.Infrastructure.GetEventStore/Formatter.cs

@@ -8,6 +8,7 @@
 
 using System.Text;
 using EventStore.ClientAPI;
+using Squidex.Infrastructure.CQRS.Events;
 using EventData = Squidex.Infrastructure.CQRS.Events.EventData;
 using EventStoreData = EventStore.ClientAPI.EventData;
 
@@ -15,12 +16,16 @@ namespace Squidex.Infrastructure.GetEventStore
 {
     public static class Formatter
     {
-        public static EventData Read(RecordedEvent eventData)
+        public static StoredEvent Read(ResolvedEvent resolvedEvent)
         {
-            var body = Encoding.UTF8.GetString(eventData.Data);
+            var @event = resolvedEvent.Event;
-            var meta = Encoding.UTF8.GetString(eventData.Metadata);
 
-            return new EventData { Type = eventData.EventType, EventId = eventData.EventId, Payload = body, Metadata = meta };
+            var body = Encoding.UTF8.GetString(@event.Data);
+            var meta = Encoding.UTF8.GetString(@event.Metadata);
+
+            var eventData = new EventData { Type = @event.EventType, EventId = @event.EventId, Payload = body, Metadata = meta };
+
+            return new StoredEvent(resolvedEvent.OriginalEventNumber.ToString(), resolvedEvent.Event.EventNumber, eventData);
         }
 
         public static EventStoreData Write(EventData eventData)

src/Squidex.Infrastructure.GetEventStore/GetEventStore.cs

@@ -71,9 +71,9 @@ namespace Squidex.Infrastructure.GetEventStore
 
                     foreach (var resolved in currentSlice.Events)
                     {
-                        var eventData = Formatter.Read(resolved.Event);
+                        var storedEvent = Formatter.Read(resolved);
 
-                        result.Add(new StoredEvent(resolved.OriginalPosition.ToString(), resolved.Event.EventNumber, eventData));
+                        result.Add(storedEvent);
                     }
                 }
             }

src/Squidex.Infrastructure.GetEventStore/GetEventStoreSubscription.cs

@@ -11,7 +11,6 @@ using System.Collections.Concurrent;
 using System.Linq;
 using System.Net;
 using System.Net.Sockets;
-using System.Text;
 using System.Threading.Tasks;
 using EventStore.ClientAPI;
 using EventStore.ClientAPI.Exceptions;
@@ -39,7 +38,7 @@ namespace Squidex.Infrastructure.GetEventStore
             this.streamFilter = streamFilter;
             this.projectionHost = projectionHost;
 
-            streamName = CreateStreamName(streamFilter, prefix);
+            streamName = $"by-{prefix.Simplify()}-{streamFilter.Simplify()}";
         }
        
         public void Dispose()
@@ -47,15 +46,40 @@ namespace Squidex.Infrastructure.GetEventStore
             internalSubscription?.Stop();
         }
 
-        public async Task SubscribeAsync(Func<StoredEvent, Task> handler)
+        public async Task SubscribeAsync(Func<StoredEvent, Task> onNext, Func<Exception, Task> onError = null)
         {
-            Guard.NotNull(handler, nameof(handler));
+            Guard.NotNull(onNext, nameof(onNext));
 
             if (internalSubscription != null)
             {
                 throw new InvalidOperationException("An handler has already been registered.");
             }
 
+            await CreateProjectionAsync();
+
+            long? eventStorePosition = null;
+
+            if (long.TryParse(position, out var parsedPosition))
+            {
+                eventStorePosition = parsedPosition;
+            }
+
+            internalSubscription = connection.SubscribeToStreamFrom(streamName, eventStorePosition, CatchUpSubscriptionSettings.Default, 
+                (subscription, resolved) =>
+                {
+                    var storedEvent = Formatter.Read(resolved);
+
+                    onNext(storedEvent).Wait();
+                }, subscriptionDropped: (subscription, reason, ex) =>
+                {
+                    var exception = ex ?? new ConnectionClosedException($"Subscription closed with reason {reason}.");
+
+                    onError?.Invoke(exception);
+                });
+        }
+
+        private async Task CreateProjectionAsync()
+        {
             if (subscriptionsCreated.TryAdd(streamName, true))
             {
                 var projectsManager = await ConnectToProjections();
@@ -79,20 +103,6 @@ namespace Squidex.Infrastructure.GetEventStore
                     // Projection already exists.
                 }
             }
-
-            long? eventStorePosition = null;
-
-            if (long.TryParse(position, out var parsedPosition))
-            {
-                eventStorePosition = parsedPosition;
-            }
-
-            internalSubscription = connection.SubscribeToStreamFrom(streamName, eventStorePosition, CatchUpSubscriptionSettings.Default, (subscription, resolved) =>
-            {
-                var eventData = Formatter.Read(resolved.Event);
-
-                handler(new StoredEvent(resolved.OriginalEventNumber.ToString(), resolved.Event.EventNumber, eventData)).Wait();
-            });
         }
 
         private async Task<ProjectionsManager> ConnectToProjections()
@@ -113,34 +123,5 @@ namespace Squidex.Infrastructure.GetEventStore
                     connection.Settings.OperationTimeout);
             return projectsManager;
         }
-
-        private static string CreateStreamName(string streamFilter, string prefix)
-        {
-            var sb = new StringBuilder();
-
-            sb.Append("by-");
-            sb.Append(prefix.Trim(' ', '-'));
-            sb.Append("-");
-
-            var prevIsLetterOrDigit = false;
-
-            foreach (var c in streamFilter)
-            {
-                if (char.IsLetterOrDigit(c))
-                {
-                    sb.Append(char.ToLowerInvariant(c));
-
-                    prevIsLetterOrDigit = true;
-                }
-                else if (prevIsLetterOrDigit)
-                {
-                    sb.Append("-");
-
-                    prevIsLetterOrDigit = false;
-                }
-            }
-
-            return sb.ToString().Trim(' ', '-');
-        }
     }
 }

src/Squidex.Infrastructure.MongoDb/EventStore/PollingSubscription.cs

@@ -38,9 +38,9 @@ namespace Squidex.Infrastructure.MongoDb.EventStore
             }
         }
 
-        public Task SubscribeAsync(Func<StoredEvent, Task> handler)
+        public Task SubscribeAsync(Func<StoredEvent, Task> onNext, Func<Exception, Task> onError = null)
         {
-            Guard.NotNull(handler, nameof(handler));
+            Guard.NotNull(onNext, nameof(onNext));
 
             if (timer == null)
             {
@@ -49,7 +49,14 @@ namespace Squidex.Infrastructure.MongoDb.EventStore
 
             timer = new CompletionTimer(5000, async ct =>
             {
-                await eventStore.GetEventsAsync(handler, ct, streamFilter, position);
+                try
+                {
+                    await eventStore.GetEventsAsync(onNext, ct, streamFilter, position);
+                }
+                catch (Exception ex)
+                {
+                    onError?.Invoke(ex);
+                }
             });
 
             eventNotifier.Subscribe(timer.Wakeup);

src/Squidex.Infrastructure/CQRS/Events/EventReceiver.cs

@@ -130,7 +130,6 @@ namespace Squidex.Infrastructure.CQRS.Events
                 catch (Exception ex)
                 {
                     log.LogFatal(ex, w => w.WriteProperty("action", "EventHandlingFailed"));
-
                 }
             });
         }
@@ -141,6 +140,13 @@ namespace Squidex.Infrastructure.CQRS.Events
 
             var subscription = eventStore.CreateSubscription(eventConsumer.EventsFilter, position);
 
+            async Task StopSubscriptionAsync(Exception exception)
+            {
+                await eventConsumerInfoRepository.StopAsync(consumerName, exception.ToString());
+
+                subscription.Dispose();
+            }
+
             await subscription.SubscribeAsync(async storedEvent =>
             {
                 try
@@ -151,11 +157,9 @@ namespace Squidex.Infrastructure.CQRS.Events
                 }
                 catch (Exception ex)
                 {
-                    await eventConsumerInfoRepository.StopAsync(consumerName, ex.ToString());
+                    await StopSubscriptionAsync(ex);
-
-                    subscription.Dispose();
                 }
-            });
+            }, StopSubscriptionAsync);
 
             currentSubscription = subscription;
         }

src/Squidex.Infrastructure/CQRS/Events/IEventSubscription.cs

@@ -13,6 +13,6 @@ namespace Squidex.Infrastructure.CQRS.Events
 {
     public interface IEventSubscription : IDisposable
     {
-        Task SubscribeAsync(Func<StoredEvent, Task> handler);
+        Task SubscribeAsync(Func<StoredEvent, Task> onNext, Func<Exception, Task> onError = null);
     }
 }

src/Squidex.Infrastructure/StringExtensions.cs

@@ -7,7 +7,9 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using System.Text.RegularExpressions;
 
 namespace Squidex.Infrastructure
@@ -16,6 +18,296 @@ namespace Squidex.Infrastructure
     {
         private static readonly Regex SlugRegex = new Regex("^[a-z0-9]+(\\-[a-z0-9]+)*$", RegexOptions.Compiled);
         private static readonly Regex PropertyNameRegex = new Regex("^[a-zA-Z0-9]+(\\-[a-zA-Z0-9]+)*$", RegexOptions.Compiled);
+        private static readonly Dictionary<char, string> LowerCaseDiacritics;
+        private static readonly Dictionary<char, string> Diacritics = new Dictionary<char, string>
+        {
+            ['À'] = "A",
+            ['à'] = "a",
+            ['Ā'] = "A",
+            ['Ġ'] = "G",
+            ['ŀ'] = "l",
+            ['Š'] = "S",
+            ['Ǡ'] = "A",
+            ['Ȁ'] = "A",
+            ['Á'] = "A",
+            ['á'] = "a",
+            ['ā'] = "a",
+            ['ġ'] = "g",
+            ['Ł'] = "L",
+            ['š'] = "s",
+            ['ǡ'] = "a",
+            ['ȁ'] = "a",
+            ['Â'] = "A",
+            ['â'] = "a",
+            ['Ă'] = "A",
+            ['Ģ'] = "G",
+            ['ł'] = "l",
+            ['Ţ'] = "T",
+            ['Ǣ'] = "Ae",
+            ['Ȃ'] = "A",
+            ['Ã'] = "A",
+            ['ã'] = "a",
+            ['ă'] = "a",
+            ['ģ'] = "g",
+            ['Ń'] = "N",
+            ['ţ'] = "t",
+            ['ǣ'] = "ae",
+            ['ȃ'] = "a",
+            ['Ä'] = "Ae",
+            ['ä'] = "ae",
+            ['Ą'] = "A",
+            ['Ĥ'] = "H",
+            ['ń'] = "n",
+            ['Ť'] = "T",
+            ['DŽ'] = "DZ",
+            ['Ǥ'] = "G",
+            ['Ȅ'] = "E",
+            ['Å'] = "A",
+            ['å'] = "a",
+            ['ą'] = "a",
+            ['ĥ'] = "h",
+            ['Ņ'] = "N",
+            ['ť'] = "t",
+            ['Dž'] = "Dz",
+            ['ǥ'] = "g",
+            ['ȅ'] = "e",
+            ['Æ'] = "AE",
+            ['æ'] = "ae",
+            ['Ć'] = "C",
+            ['Ħ'] = "H",
+            ['ņ'] = "n",
+            ['Ŧ'] = "T",
+            ['dž'] = "dz",
+            ['Ǧ'] = "G",
+            ['Ȇ'] = "E",
+            ['Ç'] = "C",
+            ['ç'] = "c",
+            ['ć'] = "c",
+            ['ħ'] = "h",
+            ['Ň'] = "N",
+            ['ŧ'] = "t",
+            ['LJ'] = "W",
+            ['ǧ'] = "g",
+            ['ȇ'] = "e",
+            ['È'] = "E",
+            ['è'] = "E",
+            ['Ĉ'] = "C",
+            ['Ĩ'] = "I",
+            ['ň'] = "n",
+            ['Ũ'] = "U",
+            ['Lj'] = "Lj",
+            ['Ǩ'] = "K",
+            ['Ȉ'] = "I",
+            ['É'] = "E",
+            ['é'] = "e",
+            ['ĉ'] = "c",
+            ['ĩ'] = "i",
+            ['ʼn'] = "n",
+            ['ũ'] = "u",
+            ['lj'] = "lj",
+            ['ǩ'] = "k",
+            ['ȉ'] = "i",
+            ['Ê'] = "E",
+            ['ê'] = "e",
+            ['Ċ'] = "C",
+            ['Ī'] = "I",
+            ['Ŋ'] = "n",
+            ['Ū'] = "U",
+            ['NJ'] = "NJ",
+            ['Ǫ'] = "O",
+            ['Ȋ'] = "I",
+            ['Ë'] = "E",
+            ['ë'] = "e",
+            ['ċ'] = "c",
+            ['ī'] = "i",
+            ['ŋ'] = "n",
+            ['ū'] = "u",
+            ['Nj'] = "Nj",
+            ['ǫ'] = "o",
+            ['ȋ'] = "i",
+            ['Ì'] = "I",
+            ['ì'] = "i",
+            ['Č'] = "C",
+            ['Ĭ'] = "I",
+            ['Ō'] = "O",
+            ['Ŭ'] = "U",
+            ['nj'] = "nj",
+            ['Ǭ'] = "O",
+            ['Ȍ'] = "O",
+            ['Í'] = "I",
+            ['í'] = "i",
+            ['č'] = "c",
+            ['ĭ'] = "i",
+            ['ō'] = "o",
+            ['ŭ'] = "u",
+            ['Ǎ'] = "A",
+            ['ǭ'] = "o",
+            ['ȍ'] = "o",
+            ['Î'] = "I",
+            ['î'] = "i",
+            ['Ď'] = "D",
+            ['Į'] = "I",
+            ['Ŏ'] = "O",
+            ['Ů'] = "U",
+            ['ǎ'] = "a",
+            ['Ǯ'] = "z",
+            ['Ȏ'] = "O",
+            ['Ï'] = "I",
+            ['ï'] = "i",
+            ['ď'] = "d",
+            ['į'] = "i",
+            ['ŏ'] = "o",
+            ['ů'] = "u",
+            ['Ǐ'] = "I",
+            ['ǯ'] = "z",
+            ['ȏ'] = "o",
+            ['Ð'] = "D",
+            ['ð'] = "d",
+            ['Đ'] = "D",
+            ['İ'] = "I",
+            ['Ő'] = "O",
+            ['Ű'] = "U",
+            ['ǐ'] = "i",
+            ['ǰ'] = "j",
+            ['Ȑ'] = "R",
+            ['Ñ'] = "N",
+            ['ñ'] = "n",
+            ['đ'] = "d",
+            ['ı'] = "i",
+            ['ő'] = "o",
+            ['ű'] = "u",
+            ['Ǒ'] = "O",
+            ['DZ'] = "DZ",
+            ['ȑ'] = "r",
+            ['Ò'] = "O",
+            ['ò'] = "o",
+            ['Ē'] = "E",
+            ['IJ'] = "LJ",
+            ['Œ'] = "OE",
+            ['Ų'] = "U",
+            ['ǒ'] = "o",
+            ['Dz'] = "Dz",
+            ['Ȓ'] = "R",
+            ['Ó'] = "O",
+            ['ó'] = "o",
+            ['ē'] = "e",
+            ['ij'] = "ij",
+            ['œ'] = "oe",
+            ['ų'] = "u",
+            ['Ǔ'] = "U",
+            ['dz'] = "dz",
+            ['ȓ'] = "r",
+            ['Ô'] = "O",
+            ['ô'] = "o",
+            ['Ĕ'] = "E",
+            ['Ĵ'] = "J",
+            ['Ŕ'] = "R",
+            ['Ŵ'] = "W",
+            ['ǔ'] = "u",
+            ['Ǵ'] = "G",
+            ['Ȕ'] = "U",
+            ['Õ'] = "O",
+            ['õ'] = "o",
+            ['ĕ'] = "e",
+            ['ĵ'] = "j",
+            ['ŕ'] = "r",
+            ['ŵ'] = "w",
+            ['Ǖ'] = "U",
+            ['ǵ'] = "g",
+            ['ȕ'] = "u",
+            ['Ö'] = "Oe",
+            ['ö'] = "oe",
+            ['Ė'] = "E",
+            ['Ķ'] = "K",
+            ['Ŗ'] = "R",
+            ['Ŷ'] = "Y",
+            ['ǖ'] = "u",
+            ['Ƕ'] = "Hj",
+            ['Ȗ'] = "U",
+            ['ė'] = "e",
+            ['ķ'] = "k",
+            ['ŗ'] = "r",
+            ['ŷ'] = "y",
+            ['Ǘ'] = "U",
+            ['ȗ'] = "u",
+            ['Ø'] = "O",
+            ['ø'] = "o",
+            ['Ę'] = "E",
+            ['ĸ'] = "k",
+            ['Ř'] = "R",
+            ['Ÿ'] = "Y",
+            ['ǘ'] = "u",
+            ['Ǹ'] = "N",
+            ['Ș'] = "S",
+            ['Ù'] = "U",
+            ['ù'] = "u",
+            ['ę'] = "e",
+            ['Ĺ'] = "L",
+            ['ř'] = "r",
+            ['Ź'] = "Z",
+            ['Ǚ'] = "U",
+            ['ǹ'] = "n",
+            ['ș'] = "s",
+            ['Ú'] = "U",
+            ['ú'] = "u",
+            ['Ě'] = "E",
+            ['ĺ'] = "l",
+            ['Ś'] = "S",
+            ['ź'] = "z",
+            ['ǚ'] = "u",
+            ['Ǻ'] = "A",
+            ['Ț'] = "T",
+            ['Û'] = "U",
+            ['û'] = "u",
+            ['ě'] = "e",
+            ['Ļ'] = "L",
+            ['ś'] = "s",
+            ['Ż'] = "Z",
+            ['Ǜ'] = "U",
+            ['ǻ'] = "a",
+            ['ț'] = "t",
+            ['Ü'] = "Ue",
+            ['ü'] = "ue",
+            ['Ĝ'] = "G",
+            ['ļ'] = "l",
+            ['Ŝ'] = "S",
+            ['ż'] = "z",
+            ['ǜ'] = "u",
+            ['Ǽ'] = "AE",
+            ['Ȝ'] = "z",
+            ['Ý'] = "Y",
+            ['ý'] = "y",
+            ['ĝ'] = "g",
+            ['Ľ'] = "L",
+            ['ŝ'] = "s",
+            ['Ž'] = "Z",
+            ['ǝ'] = "e",
+            ['ǽ'] = "ae",
+            ['ȝ'] = "z",
+            ['Þ'] = "p",
+            ['þ'] = "p",
+            ['Ğ'] = "G",
+            ['ľ'] = "L",
+            ['Ş'] = "S",
+            ['ž'] = "z",
+            ['Ǟ'] = "A",
+            ['Ǿ'] = "O",
+            ['Ȟ'] = "H",
+            ['ß'] = "ss",
+            ['ÿ'] = "y",
+            ['ğ'] = "g",
+            ['Ŀ'] = "L",
+            ['ş'] = "s",
+            ['ſ'] = "l",
+            ['ǟ'] = "a",
+            ['ǿ'] = "o",
+            ['ȟ'] = "h"
+        };
+
+        static StringExtensions()
+        {
+            LowerCaseDiacritics = Diacritics.ToDictionary(x => x.Key, x => x.Value.ToLowerInvariant());
+        }
 
         public static bool IsSlug(this string value)
         {
@@ -43,5 +335,52 @@ namespace Squidex.Infrastructure
 
             return char.ToLower(value[0]) + value.Substring(1);
         }
+
+        public static string Simplify(this string value, ISet<char> preserveHash = null, bool singleCharDiactric = false, char separator = '-')
+        {
+            var result = new StringBuilder(value.Length);
+
+            var lastChar = (char)0;
+
+            for (var i = 0; i < value.Length; i++)
+            {
+                var character = value[i];
+
+                if (preserveHash?.Contains(character) == true)
+                {
+                    result.Append(character);
+                }
+                else if (char.IsLetter(character) || char.IsNumber(character))
+                {
+                    lastChar = character;
+
+                    var lower = char.ToLowerInvariant(character);
+
+                    if (LowerCaseDiacritics.TryGetValue(character, out string replacement))
+                    {
+                        if (singleCharDiactric)
+                        {
+                            result.Append(replacement[0]);
+                        }
+                        else
+                        {
+                            result.Append(replacement);
+                        }
+                    }
+                    else
+                    {
+                        result.Append(lower);
+                    }
+                }
+                else if ((i < value.Length - 1) && (i > 0 && lastChar != separator))
+                {
+                    lastChar = separator;
+
+                    result.Append(separator);
+                }
+            }
+
+            return result.ToString().Trim(separator);
+        }
     }
 }

tests/Squidex.Infrastructure.Tests/CQRS/Events/EventReceiverTests.cs

@@ -43,7 +43,7 @@ namespace Squidex.Infrastructure.CQRS.Events
                 this.storedEvents = storedEvents;
             }
 
-            public Task SubscribeAsync(Func<StoredEvent, Task> handler)
+            public Task SubscribeAsync(Func<StoredEvent, Task> onNext, Func<Exception, Task> onError)
             {
                 foreach (var storedEvent in storedEvents)
                 {
@@ -52,7 +52,7 @@ namespace Squidex.Infrastructure.CQRS.Events
                         break;
                     }
 
-                    handler(storedEvent).Wait();
+                    onNext(storedEvent).Wait();
                 }
 
                 return TaskHelper.Done;

tests/Squidex.Infrastructure.Tests/StringExtensionsTests.cs

@@ -6,12 +6,22 @@
 //  All rights reserved.
 // ==========================================================================
 
+using System.Collections.Generic;
 using Xunit;
 
 namespace Squidex.Infrastructure
 {
     public class StringExtensionsTests
     {
+        [Theory]
+        [InlineData(null)]
+        [InlineData("")]
+        [InlineData(" ")]
+        public void Should_provide_fallback_if_invalid(string value)
+        {
+            Assert.Equal("fallback", value.WithFallback("fallback"));
+        }
+
         [Theory]
         [InlineData("my", "My")]
         [InlineData("myProperty ", "MyProperty")]
@@ -35,12 +45,55 @@ namespace Squidex.Infrastructure
         }
 
         [Theory]
-        [InlineData(null)]
+        [InlineData("Hello World", '-', "hello-world")]
-        [InlineData("")]
+        [InlineData("Hello/World", '-', "hello-world")]
-        [InlineData(" ")]
+        [InlineData("Hello World", '_', "hello_world")]
-        public void Should_provide_fallback_if_invalid(string value)
+        [InlineData("Hello/World", '_', "hello_world")]
+        [InlineData("Hello World ", '_', "hello_world")]
+        [InlineData("Hello World-", '_', "hello_world")]
+        [InlineData("Hello/World_", '_', "hello_world")]
+        public void Should_replace_special_characters_with_sepator_when_simplifying(string input, char separator, string output)
         {
-            Assert.Equal("fallback", value.WithFallback("fallback"));
+            Assert.Equal(output, input.Simplify(separator: separator));
+        }
+
+        [Theory]
+        [InlineData("ö", "oe")]
+        [InlineData("ü", "ue")]
+        [InlineData("ä", "ae")]
+        public void Should_replace_multi_char_diacritics_when_simplifying(string input, string output)
+        {
+            Assert.Equal(output, input.Simplify());
+        }
+
+        [Theory]
+        [InlineData("ö", "o")]
+        [InlineData("ü", "u")]
+        [InlineData("ä", "a")]
+        public void Should_not_replace_multi_char_diacritics_when_simplifying(string input, string output)
+        {
+            Assert.Equal(output, input.Simplify(singleCharDiactric: true));
+        }
+
+        [Theory]
+        [InlineData("Físh", "fish")]
+        [InlineData("źish", "zish")]
+        [InlineData("żish", "zish")]
+        [InlineData("fórm", "form")]
+        [InlineData("fòrm", "form")]
+        [InlineData("fårt", "fart")]
+        public void Should_replace_single_char_diacritics_when_simplifying(string input, string output)
+        {
+            Assert.Equal(output, input.Simplify());
+        }
+
+        [Theory]
+        [InlineData("Hello my&World ", '_', "hello_my&world")]
+        [InlineData("Hello my&World-", '_', "hello_my&world")]
+        [InlineData("Hello my/World_", '_', "hello_my/world")]
+        public void Should_keep_characters_when_simplifying(string input, char separator, string output)
+        {
+            Assert.Equal(output, input.Simplify(new HashSet<char> { '&', '/' }, false, separator));
         }
 
         [Fact]