Introduce line break tests

src/Avalonia.Visuals/Media/TextFormatting/Unicode/BreakPairTable.cs

@@ -5,8 +5,8 @@ namespace Avalonia.Media.TextFormatting.Unicode
         private static readonly byte[][] s_breakPairTable = 
             {
              new byte[] {4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,4,4,4,4,4,4,4,4,4,4,4},
-             new byte[] {0,4,4,1,1,4,4,4,4,1,1,0,0,0,0,0,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0},
+             new byte[] {0,4,4,1,1,4,4,4,4,1,1,0,0,0,0,4,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0},
-             new byte[] {0,4,4,1,1,4,4,4,4,1,1,1,1,1,0,0,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0},
+             new byte[] {0,4,4,1,1,4,4,4,4,1,1,1,1,1,0,4,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0},
              new byte[] {4,4,4,1,1,1,4,4,4,1,1,1,1,1,1,1,1,1,1,1,4,2,4,1,1,1,1,1,1,1,1,1,1},
              new byte[] {1,4,4,1,1,1,4,4,4,1,1,1,1,1,1,1,1,1,1,1,4,2,4,1,1,1,1,1,1,1,1,1,1},
              new byte[] {0,4,4,1,1,1,4,4,4,0,0,0,0,0,0,0,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0},

src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakClass.cs

@@ -34,8 +34,8 @@ namespace Avalonia.Media.TextFormatting.Unicode
         EBase, //EB
         EModifier, //EM
         ZWJ, //ZWJ
-
         ContingentBreak, //CB
+
         Unknown, //XX
         Ambiguous, //AI
         MandatoryBreak, //BK

src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakEnumerator.cs

@@ -95,6 +95,7 @@ namespace Avalonia.Media.TextFormatting.Unicode
 
                     if (_nextClass.Value == LineBreakClass.MandatoryBreak)
                     {
+                        _lastPos = _pos;
                         Current = new LineBreak(FindPriorNonWhitespace(_lastPos), _lastPos);
                         return true;
                     }
@@ -108,6 +109,7 @@ namespace Avalonia.Media.TextFormatting.Unicode
                 {
                     case PairBreakType.DI: // Direct break
                         shouldBreak = true;
+                        _lastPos = _pos;
                         break;
 
                     case PairBreakType.IN: // possible indirect break

tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/BreakPairTable.txt

@@ -1,7 +1,7 @@
 	OP	CL	CP	QU	GL	NS	EX	SY	IS	PR	PO	NU	AL	HL	ID	IN	HY	BA	BB	B2	ZW	CM	WJ	H2	H3	JL	JV	JT	RI	EB	EM	ZWJ	CB
 OP	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	^	@	^	^	^	^	^	^	^	^	^	^	^
-CL	_	^	^	%	%	^	^	^	^	%	%	_	_	_	_	_	%	%	_	_	^	#	^	_	_	_	_	_	_	_	_	%	_
+CL	_	^	^	%	%	^	^	^	^	%	%	_	_	_	_	^	%	%	_	_	^	#	^	_	_	_	_	_	_	_	_	%	_
-CP	_	^	^	%	%	^	^	^	^	%	%	%	%	%	_	_	%	%	_	_	^	#	^	_	_	_	_	_	_	_	_	%	_
+CP	_	^	^	%	%	^	^	^	^	%	%	%	%	%	_	^	%	%	_	_	^	#	^	_	_	_	_	_	_	_	_	%	_
 QU	^	^	^	%	%	%	^	^	^	%	%	%	%	%	%	%	%	%	%	%	^	#	^	%	%	%	%	%	%	%	%	%	%
 GL	%	^	^	%	%	%	^	^	^	%	%	%	%	%	%	%	%	%	%	%	^	#	^	%	%	%	%	%	%	%	%	%	%
 NS	_	^	^	%	%	%	^	^	^	_	_	_	_	_	_	_	%	%	_	_	^	#	^	_	_	_	_	_	_	_	_	%	_

tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/GraphemeBreakClassTrieGeneratorTests.cs

@@ -1,9 +1,4 @@
 using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
 using Avalonia.Media.TextFormatting.Unicode;
 using Xunit;
 
@@ -16,7 +11,7 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
     public class GraphemeBreakClassTrieGeneratorTests
     {
         [Theory/*(Skip = "Only run when we update the trie.")*/]
-        [ClassData(typeof(GraphemeEnumeratorTestDataGenerator))]
+        [ClassData(typeof(GraphemeBreakTestDataGenerator))]
         public void Should_Enumerate(string text, int expectedLength)
         {
             var textMemory = text.AsMemory();
@@ -55,77 +50,11 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
             GraphemeBreakClassTrieGenerator.Execute();
         }
 
-        public class GraphemeEnumeratorTestDataGenerator : IEnumerable<object[]>
+        private class GraphemeBreakTestDataGenerator : TestDataGenerator
         {
-            private readonly List<object[]> _testData;
+            public GraphemeBreakTestDataGenerator() 
-
+                : base("auxiliary/GraphemeBreakTest.txt")
-            public GraphemeEnumeratorTestDataGenerator()
-            {
-                _testData = ReadTestData();
-            }
-
-            public IEnumerator<object[]> GetEnumerator()
-            {
-                return _testData.GetEnumerator();
-            }
-
-            IEnumerator IEnumerable.GetEnumerator()
-            {
-                return GetEnumerator();
-            }
-
-            private static List<object[]> ReadTestData()
             {
-                var testData = new List<object[]>();
-
-                using (var client = new HttpClient())
-                {
-                    var url = Path.Combine(UnicodeDataGenerator.Ucd, "auxiliary/GraphemeBreakTest.txt");
-
-                    using (var result = client.GetAsync(url).GetAwaiter().GetResult())
-                    {
-                        if (!result.IsSuccessStatusCode)
-                            return testData;
-
-                        using (var stream = result.Content.ReadAsStreamAsync().GetAwaiter().GetResult())
-                        using (var reader = new StreamReader(stream))
-                        {
-                            while (!reader.EndOfStream)
-                            {
-                                var line = reader.ReadLine();
-
-                                if (line == null)
-                                {
-                                    break;
-                                }
-
-                                if (line.StartsWith("#") || string.IsNullOrEmpty(line))
-                                {
-                                    continue;
-                                }
-
-                                var elements = line.Split('#');
-
-                                elements = elements[0].Replace("÷\t", "÷").Trim('÷').Split('÷');
-
-                                var chars = elements[0].Replace(" × ", " ").Split(' ');
-
-                                var codepoints = chars.Where(x => x != "" && x != "×")
-                                    .Select(x => Convert.ToInt32(x, 16)).ToArray();
-
-                                var text = string.Join(null, codepoints.Select(char.ConvertFromUtf32));
-
-                                var length = codepoints.Select(x => x > ushort.MaxValue ? 2 : 1).Sum();
-
-                                var data = new object[] { text, length };
-
-                                testData.Add(data);
-                            }
-                        }
-                    }
-                }
-
-                return testData;
             }
         }
     }

tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/TestDataGenerator.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+
+namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
+{
+    public abstract class TestDataGenerator : IEnumerable<object[]>
+    {
+        private readonly string _fileName;
+        private readonly List<object[]> _testData;
+
+        protected TestDataGenerator(string fileName)
+        {
+            _fileName = fileName;
+            _testData = ReadTestData();
+        }
+
+        public IEnumerator<object[]> GetEnumerator()
+        {
+            return _testData.GetEnumerator();
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        private List<object[]> ReadTestData()
+        {
+            var testData = new List<object[]>();
+
+            using (var client = new HttpClient())
+            {
+                var url = Path.Combine(UnicodeDataGenerator.Ucd, _fileName);
+
+                using (var result = client.GetAsync(url).GetAwaiter().GetResult())
+                {
+                    if (!result.IsSuccessStatusCode)
+                        return testData;
+
+                    using (var stream = result.Content.ReadAsStreamAsync().GetAwaiter().GetResult())
+                    using (var reader = new StreamReader(stream))
+                    {
+                        while (!reader.EndOfStream)
+                        {
+                            var line = reader.ReadLine();
+
+                            if (line == null)
+                            {
+                                break;
+                            }
+
+                            if (line.StartsWith("#") || string.IsNullOrEmpty(line))
+                            {
+                                continue;
+                            }
+
+                            var elements = line.Split('#');
+
+                            elements = elements[0].Replace("÷\t", "÷").Trim('÷').Split('÷');
+
+                            var chars = elements[0].Replace(" × ", " ").Split(' ');
+
+                            var codepoints = chars.Where(x => x != "" && x != "×")
+                                .Select(x => Convert.ToInt32(x, 16)).ToArray();
+
+                            var text = string.Join(null, codepoints.Select(char.ConvertFromUtf32));
+
+                            var length = codepoints.Select(x => x > ushort.MaxValue ? 2 : 1).Sum();
+
+                            var data = new object[] { text, length };
+
+                            testData.Add(data);
+                        }
+                    }
+                }
+            }
+
+            return testData;
+        }
+    }
+}

tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeDataGeneratorTests.cs

@@ -1,4 +1,6 @@
-using Xunit;
+using System;
+using Avalonia.Media.TextFormatting.Unicode;
+using Xunit;
 
 namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
 {
@@ -13,5 +15,26 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
         {
             UnicodeDataGenerator.Execute();
         }
+        [Theory/*(Skip = "Only run when we update the trie.")*/]
+        [ClassData(typeof(LineBreakTestDataGenerator))]
+
+        public void Should_Enumerate_LineBreaks(string text, int expectedLength)
+        {
+            var textMemory = text.AsMemory();
+
+            var enumerator = new LineBreakEnumerator(textMemory);
+
+            Assert.True(enumerator.MoveNext());
+
+            Assert.Equal(expectedLength, enumerator.Current.PositionWrap);
+        }
+
+        private class LineBreakTestDataGenerator : TestDataGenerator
+        {
+            public LineBreakTestDataGenerator()
+                : base("auxiliary/LineBreakTest.txt")
+            {
+            }
+        }
     }
 }

tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeEnumsGenerator.cs

@@ -42,7 +42,7 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting
 
         public static List<DataEntry> CreateGeneralCategoryEnum()
         {
-            var entries = new List<DataEntry> { new DataEntry("Other", "C#", string.Empty) };
+            var entries = new List<DataEntry> { new DataEntry("Other", "C", " Cc | Cf | Cn | Co | Cs") };
 
             ParseDataEntries("# General_Category (gc)", entries);