Support for native language names. (#809)

backend/src/Squidex.Infrastructure/Language.cs

@@ -14,34 +14,35 @@ namespace Squidex.Infrastructure
     [TypeConverter(typeof(LanguageTypeConverter))]
     public partial record Language
     {
-#pragma warning disable MA0023 // Add RegexOptions.ExplicitCapture
-        private static readonly Regex CultureRegex = new Regex("^([a-z]{2})(\\-[a-z]{2})?$", RegexOptions.IgnoreCase);
-#pragma warning restore MA0023 // Add RegexOptions.ExplicitCapture
+        private static readonly Regex CultureRegex = new Regex("^(?<Code>[a-z]{2})(\\-[a-z]{2})?$", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture);
 
         public static Language GetLanguage(string iso2Code)
         {
             Guard.NotNullOrEmpty(iso2Code, nameof(iso2Code));
 
-            try
-            {
-                return AllLanguagesField[iso2Code];
-            }
-            catch (KeyNotFoundException)
+            if (!LanguageByCode.TryGetValue(iso2Code, out var result))
             {
                 throw new NotSupportedException($"Language {iso2Code} is not supported");
             }
+
+            return result;
         }
 
         public static IReadOnlyCollection<Language> AllLanguages
         {
-            get => AllLanguagesField.Values;
+            get => LanguageByCode.Values;
         }
 
         public string Iso2Code { get; }
 
         public string EnglishName
         {
-            get => AllLanguagesNames.GetOrDefault(Iso2Code) ?? string.Empty;
+            get => NamesEnglish.GetOrDefault(Iso2Code) ?? string.Empty;
+        }
+
+        public string NativeName
+        {
+            get => NamesNative.GetOrDefault(Iso2Code) ?? string.Empty;
         }
 
         private Language(string iso2Code)
@@ -53,14 +54,14 @@ namespace Squidex.Infrastructure
         {
             Guard.NotNull(iso2Code, nameof(iso2Code));
 
-            return AllLanguagesField.ContainsKey(iso2Code);
+            return LanguageByCode.ContainsKey(iso2Code);
         }
 
         public static bool TryGetLanguage(string iso2Code, [MaybeNullWhen(false)] out Language language)
         {
             Guard.NotNull(iso2Code, nameof(iso2Code));
 
-            return AllLanguagesField.TryGetValue(iso2Code, out language!);
+            return LanguageByCode.TryGetValue(iso2Code, out language!);
         }
 
         public static implicit operator string(Language language)
@@ -91,7 +92,7 @@ namespace Squidex.Infrastructure
                     return null;
                 }
 
-                input = match.Groups[1].Value;
+                input = match.Groups["Code"].Value;
             }
 
             if (TryGetLanguage(input.ToLowerInvariant(), out var result))

backend/src/Squidex.Infrastructure/LanguagesInitializer.cs

@@ -30,7 +30,7 @@ namespace Squidex.Infrastructure
                 {
                     if (!Language.TryGetLanguage(key, out _))
                     {
-                        Language.AddLanguage(key, value);
+                        Language.AddLanguage(key, value, value);
                     }
                 }
             }

backend/src/Squidex/Areas/Api/Controllers/LanguageDto.cs

@@ -25,6 +25,12 @@ namespace Squidex.Areas.Api.Controllers
         [LocalizedRequired]
         public string EnglishName { get; set; }
 
+        /// <summary>
+        /// The native name of the language.
+        /// </summary>
+        [LocalizedRequired]
+        public string NativeName { get; set; }
+
         public static LanguageDto FromLanguage(Language language)
         {
             return SimpleMapper.Map(language, new LanguageDto());

backend/tools/GenerateLanguages/GenerateLanguages.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+	<TargetFramework>net6.0</TargetFramework>
     <LangVersion>10.0</LangVersion>
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
@@ -12,8 +12,17 @@
     </PackageReference>
     <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
+    <PackageReference Include="System.Text.Json" Version="6.0.0" />
   </ItemGroup>
   <ItemGroup>
     <AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" />
   </ItemGroup>
+  <ItemGroup>
+    <None Update="languages.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="old-languages.csv">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

backend/tools/GenerateLanguages/Language.cs

@@ -0,0 +1,20 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace GenerateLanguages
+{
+    public sealed class Language
+    {
+        public string Code { get; set; }
+
+        public string NameEnglish { get; set; }
+
+        public string NameNative { get; set; }
+
+        public string FieldName { get; set; }
+    }
+}

backend/tools/GenerateLanguages/Program.cs

@@ -7,6 +7,7 @@
 
 using System.Globalization;
 using System.Text;
+using System.Text.Json;
 
 namespace GenerateLanguages
 {
@@ -14,10 +15,11 @@ namespace GenerateLanguages
     {
         public static void Main()
         {
-            var languageCodesFile = new FileInfo("../../../../../src/Squidex.Infrastructure/language-codes.csv");
-            var languageFile = Path.Combine(languageCodesFile.DirectoryName, "Languages.cs");
+            var languageCodesFile = new FileInfo("../../../../../src/Squidex.Infrastructure/Languages.cs");
 
-            var writer = new StringWriter();
+            var languages = ReadLanguages();
+
+            var writer = new SourceBuilder();
             writer.WriteLine("// ==========================================================================");
             writer.WriteLine("//  Languages.cs");
             writer.WriteLine("//  Squidex Headless CMS");
@@ -27,99 +29,102 @@ namespace GenerateLanguages
             writer.WriteLine("// ==========================================================================");
             writer.WriteLine("// <autogenerated/>");
             writer.WriteLine();
-            writer.WriteLine("using System;");
             writer.WriteLine("using System.CodeDom.Compiler;");
-            writer.WriteLine("using System.Collections.Generic;");
             writer.WriteLine();
             writer.WriteLine("namespace Squidex.Infrastructure");
             writer.WriteLine("{");
-            writer.WriteLine("    [GeneratedCode(\"LanguagesGenerator\", \"1.0\")]");
-            writer.WriteLine("    public partial record Language");
-            writer.WriteLine("    {");
-            writer.WriteLine("        private static readonly Dictionary<string, Language> AllLanguagesField = new Dictionary<string, Language>(StringComparer.OrdinalIgnoreCase);");
-            writer.WriteLine("        private static readonly Dictionary<string, string> AllLanguagesNames = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);");
-            writer.WriteLine();
-            writer.WriteLine("        internal static Language AddLanguage(string iso2Code, string englishName)");
-            writer.WriteLine("        {");
-            writer.WriteLine("            AllLanguagesNames[iso2Code] = englishName;");
+            writer.WriteLine("[GeneratedCode(\"LanguagesGenerator\", \"1.0\")]");
+            writer.WriteLine("public partial record Language");
+            writer.WriteLine("{");
+            writer.WriteLine("private static readonly Dictionary<string, Language> LanguageByCode = new Dictionary<string, Language>(StringComparer.OrdinalIgnoreCase);");
+            writer.WriteLine("private static readonly Dictionary<string, string> NamesEnglish = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);");
+            writer.WriteLine("private static readonly Dictionary<string, string> NamesNative = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);");
             writer.WriteLine();
-            writer.WriteLine("            return AllLanguagesField.GetOrAdd(iso2Code, code => new Language(code));");
-            writer.WriteLine("        }");
+            writer.WriteLine("internal static Language AddLanguage(string iso2Code, string englishName, string nativeName)");
+            writer.WriteLine("{");
+            writer.WriteLine("NamesEnglish[iso2Code] = englishName;");
+            writer.WriteLine("NamesNative[iso2Code] = nativeName;");
             writer.WriteLine();
-
-            var languages = GetLanguages(languageCodesFile).ToList();
-
-            foreach (var (iso2Code, englishName) in languages)
-            {
-                var fieldName = iso2Code.ToUpperInvariant();
-
-                writer.WriteLine($"        public static readonly Language {fieldName} = AddLanguage(\"{iso2Code}\", \"{englishName}\");");
-            }
-
+            writer.WriteLine("return LanguageByCode.GetOrAdd(iso2Code, code => new Language(code));");
+            writer.WriteLine("}");
             writer.WriteLine();
 
-            foreach (var (code, englishName) in GetCultures(languages))
+            foreach (var language in languages)
             {
-                var fieldName = englishName.ToFieldName();
-
-                writer.WriteLine($"        public static readonly Language {fieldName} = AddLanguage(\"{code}\", \"{englishName}\");");
+                writer.WriteLine($"public static readonly Language {language.FieldName} = AddLanguage(\"{language.Code}\", \"{language.NameEnglish}\", \"{language.NameNative}\");");
             }
 
-            writer.WriteLine("    }");
+            writer.WriteLine("}");
             writer.WriteLine("}");
 
-            File.WriteAllText(languageFile, writer.ToString());
+            File.WriteAllText(languageCodesFile.FullName, writer.ToString());
         }
 
-        private static string ToFieldName(this string name)
+        private static List<Language> ReadLanguages()
         {
-            var sb = new StringBuilder();
+            var languages = new List<Language>();
+
+            var json = File.ReadAllText("languages.json");
+
+            var deserialized = JsonSerializer.Deserialize<Dictionary<string, Dictionary<string, string>>>(json);
 
-            foreach (var c in name)
+            static string ToFieldName(string name)
             {
-                if (char.IsLetterOrDigit(c))
+                var sb = new StringBuilder();
+
+                foreach (var c in name)
                 {
-                    sb.Append(c);
+                    if (char.IsLetterOrDigit(c))
+                    {
+                        sb.Append(c);
+                    }
                 }
-            }
 
-            return sb.ToString();
-        }
-
-        private static IEnumerable<(string Code, string EnglishName)> GetCultures(List<(string Iso2Code, string EnglishName)> languages)
-        {
-            return CultureInfo.GetCultures(CultureTypes.SpecificCultures)
-                .Where(x => x.ToString().Length == 5)
-                .Where(x => languages.Any(l => l.Iso2Code.Equals(x.TwoLetterISOLanguageName, StringComparison.OrdinalIgnoreCase)))
-                .Select(x => (x.ToString(), x.EnglishName));
-        }
+                return sb.ToString();
+            }
 
-        private static IEnumerable<(string Iso2Code, string EnglishName)> GetLanguages(FileInfo file)
-        {
-            var uniqueCodes = new HashSet<string>(new[] { "iv" });
+            foreach (var (code, value) in deserialized)
+            {
+                var name = value["name"];
 
-            var resourceStream = file.OpenRead();
+                languages.Add(new Language
+                {
+                    Code = code,
+                    NameNative = value["nativeName"],
+                    NameEnglish = value["name"],
+                    FieldName = code.ToUpperInvariant()
+                });
+            }
 
-            using (var reader = new StreamReader(resourceStream, Encoding.UTF8))
+            foreach (var culture in CultureInfo.GetCultures(CultureTypes.SpecificCultures))
             {
-                reader.ReadLine();
+                var code = culture.ToString();
 
-                while (!reader.EndOfStream)
+                if (code.Length == 5 && languages.Any(l => l.Code.Equals(culture.TwoLetterISOLanguageName, StringComparison.OrdinalIgnoreCase)))
                 {
-                    var line = reader.ReadLine();
+                    languages.Add(new Language
+                    {
+                        Code = code,
+                        NameNative = culture.NativeName,
+                        NameEnglish = culture.EnglishName,
+                        FieldName = ToFieldName(culture.EnglishName)
+                    });
+                }
+            }
 
-                    var iso2Code = line.Substring(1, 2);
+            var oldFile = File.ReadAllLines("old-languages.csv");
 
-                    if (uniqueCodes.Add(iso2Code))
-                    {
-                        yield return (iso2Code, line[6..^1]);
-                    }
-                    else
-                    {
-                        Console.WriteLine("Languages contains duplicate {0}", iso2Code);
-                    }
+            foreach (var line in oldFile.Skip(1))
+            {
+                var code = line.Substring(1, 2);
+
+                if (!languages.Any(x => x.Code == code))
+                {
+                    throw new InvalidOperationException($"Cannot find old language code {code}.");
                 }
             }
+
+            return languages;
         }
     }
 }

backend/tools/GenerateLanguages/SourceBuilder.cs

@@ -0,0 +1,65 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Text;
+
+namespace GenerateLanguages
+{
+    internal sealed class SourceBuilder
+    {
+        private readonly StringBuilder builder = new StringBuilder();
+        private int intentation;
+
+        public SourceBuilder StartBlock()
+        {
+            intentation += 4;
+
+            return this;
+        }
+
+        public SourceBuilder WriteLine(string line)
+        {
+            if (line.Equals("}", StringComparison.Ordinal))
+            {
+                EndBlock();
+            }
+
+            for (var i = 0; i < intentation; i++)
+            {
+                builder.Append(' ');
+            }
+
+            builder.AppendLine(line);
+
+            if (line.Equals("{", StringComparison.Ordinal))
+            {
+                StartBlock();
+            }
+
+            return this;
+        }
+
+        public SourceBuilder WriteLine()
+        {
+            builder.AppendLine();
+
+            return this;
+        }
+
+        public SourceBuilder EndBlock()
+        {
+            intentation -= 4;
+
+            return this;
+        }
+
+        public override string ToString()
+        {
+            return builder.ToString();
+        }
+    }
+}

backend/src/Squidex.Infrastructure/language-codes.csv → backend/tools/GenerateLanguages/old-languages.csv

@@ -1,4 +1,4 @@
-"alpha2","English"
+"alpha2","English"
 "aa","Afar"
 "ab","Abkhazian"
 "ae","Avestan"