From f377172947ea44a5ddeaa2cb79d47b197f2d3fbc Mon Sep 17 00:00:00 2001 From: Benedikt Schroeder Date: Tue, 30 Jun 2020 13:42:27 +0200 Subject: [PATCH] Introduce line break tests --- .../TextFormatting/Unicode/BreakPairTable.cs | 4 +- .../TextFormatting/Unicode/LineBreakClass.cs | 2 +- .../Unicode/LineBreakEnumerator.cs | 2 + .../Media/TextFormatting/BreakPairTable.txt | 4 +- .../GraphemeBreakClassTrieGeneratorTests.cs | 79 +---------------- .../Media/TextFormatting/TestDataGenerator.cs | 85 +++++++++++++++++++ .../UnicodeDataGeneratorTests.cs | 25 +++++- .../TextFormatting/UnicodeEnumsGenerator.cs | 2 +- 8 files changed, 121 insertions(+), 82 deletions(-) create mode 100644 tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/TestDataGenerator.cs diff --git a/src/Avalonia.Visuals/Media/TextFormatting/Unicode/BreakPairTable.cs b/src/Avalonia.Visuals/Media/TextFormatting/Unicode/BreakPairTable.cs index 76c94b324a..86d39a4283 100644 --- a/src/Avalonia.Visuals/Media/TextFormatting/Unicode/BreakPairTable.cs +++ b/src/Avalonia.Visuals/Media/TextFormatting/Unicode/BreakPairTable.cs @@ -5,8 +5,8 @@ namespace Avalonia.Media.TextFormatting.Unicode private static readonly byte[][] s_breakPairTable = { new byte[] {4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,4,4,4,4,4,4,4,4,4,4,4}, - new byte[] {0,4,4,1,1,4,4,4,4,1,1,0,0,0,0,0,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0}, - new byte[] {0,4,4,1,1,4,4,4,4,1,1,1,1,1,0,0,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0}, + new byte[] {0,4,4,1,1,4,4,4,4,1,1,0,0,0,0,4,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0}, + new byte[] {0,4,4,1,1,4,4,4,4,1,1,1,1,1,0,4,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0}, new byte[] {4,4,4,1,1,1,4,4,4,1,1,1,1,1,1,1,1,1,1,1,4,2,4,1,1,1,1,1,1,1,1,1,1}, new byte[] {1,4,4,1,1,1,4,4,4,1,1,1,1,1,1,1,1,1,1,1,4,2,4,1,1,1,1,1,1,1,1,1,1}, new byte[] {0,4,4,1,1,1,4,4,4,0,0,0,0,0,0,0,1,1,0,0,4,2,4,0,0,0,0,0,0,0,0,1,0}, diff --git a/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakClass.cs b/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakClass.cs index 0cf135ba21..8b2e3f41e3 100644 --- a/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakClass.cs +++ b/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakClass.cs @@ -34,8 +34,8 @@ namespace Avalonia.Media.TextFormatting.Unicode EBase, //EB EModifier, //EM ZWJ, //ZWJ - ContingentBreak, //CB + Unknown, //XX Ambiguous, //AI MandatoryBreak, //BK diff --git a/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakEnumerator.cs b/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakEnumerator.cs index a11c008409..79c140a957 100644 --- a/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakEnumerator.cs +++ b/src/Avalonia.Visuals/Media/TextFormatting/Unicode/LineBreakEnumerator.cs @@ -95,6 +95,7 @@ namespace Avalonia.Media.TextFormatting.Unicode if (_nextClass.Value == LineBreakClass.MandatoryBreak) { + _lastPos = _pos; Current = new LineBreak(FindPriorNonWhitespace(_lastPos), _lastPos); return true; } @@ -108,6 +109,7 @@ namespace Avalonia.Media.TextFormatting.Unicode { case PairBreakType.DI: // Direct break shouldBreak = true; + _lastPos = _pos; break; case PairBreakType.IN: // possible indirect break diff --git a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/BreakPairTable.txt b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/BreakPairTable.txt index 93d531c700..814ce15d0a 100644 --- a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/BreakPairTable.txt +++ b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/BreakPairTable.txt @@ -1,7 +1,7 @@ OP CL CP QU GL NS EX SY IS PR PO NU AL HL ID IN HY BA BB B2 ZW CM WJ H2 H3 JL JV JT RI EB EM ZWJ CB OP ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ @ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ -CL _ ^ ^ % % ^ ^ ^ ^ % % _ _ _ _ _ % % _ _ ^ # ^ _ _ _ _ _ _ _ _ % _ -CP _ ^ ^ % % ^ ^ ^ ^ % % % % % _ _ % % _ _ ^ # ^ _ _ _ _ _ _ _ _ % _ +CL _ ^ ^ % % ^ ^ ^ ^ % % _ _ _ _ ^ % % _ _ ^ # ^ _ _ _ _ _ _ _ _ % _ +CP _ ^ ^ % % ^ ^ ^ ^ % % % % % _ ^ % % _ _ ^ # ^ _ _ _ _ _ _ _ _ % _ QU ^ ^ ^ % % % ^ ^ ^ % % % % % % % % % % % ^ # ^ % % % % % % % % % % GL % ^ ^ % % % ^ ^ ^ % % % % % % % % % % % ^ # ^ % % % % % % % % % % NS _ ^ ^ % % % ^ ^ ^ _ _ _ _ _ _ _ % % _ _ ^ # ^ _ _ _ _ _ _ _ _ % _ diff --git a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/GraphemeBreakClassTrieGeneratorTests.cs b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/GraphemeBreakClassTrieGeneratorTests.cs index 5ae0b1448b..d7e2128f69 100644 --- a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/GraphemeBreakClassTrieGeneratorTests.cs +++ b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/GraphemeBreakClassTrieGeneratorTests.cs @@ -1,9 +1,4 @@ using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; using Avalonia.Media.TextFormatting.Unicode; using Xunit; @@ -16,7 +11,7 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting public class GraphemeBreakClassTrieGeneratorTests { [Theory/*(Skip = "Only run when we update the trie.")*/] - [ClassData(typeof(GraphemeEnumeratorTestDataGenerator))] + [ClassData(typeof(GraphemeBreakTestDataGenerator))] public void Should_Enumerate(string text, int expectedLength) { var textMemory = text.AsMemory(); @@ -55,77 +50,11 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting GraphemeBreakClassTrieGenerator.Execute(); } - public class GraphemeEnumeratorTestDataGenerator : IEnumerable + private class GraphemeBreakTestDataGenerator : TestDataGenerator { - private readonly List _testData; - - public GraphemeEnumeratorTestDataGenerator() - { - _testData = ReadTestData(); - } - - public IEnumerator GetEnumerator() - { - return _testData.GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - private static List ReadTestData() + public GraphemeBreakTestDataGenerator() + : base("auxiliary/GraphemeBreakTest.txt") { - var testData = new List(); - - using (var client = new HttpClient()) - { - var url = Path.Combine(UnicodeDataGenerator.Ucd, "auxiliary/GraphemeBreakTest.txt"); - - using (var result = client.GetAsync(url).GetAwaiter().GetResult()) - { - if (!result.IsSuccessStatusCode) - return testData; - - using (var stream = result.Content.ReadAsStreamAsync().GetAwaiter().GetResult()) - using (var reader = new StreamReader(stream)) - { - while (!reader.EndOfStream) - { - var line = reader.ReadLine(); - - if (line == null) - { - break; - } - - if (line.StartsWith("#") || string.IsNullOrEmpty(line)) - { - continue; - } - - var elements = line.Split('#'); - - elements = elements[0].Replace("÷\t", "÷").Trim('÷').Split('÷'); - - var chars = elements[0].Replace(" × ", " ").Split(' '); - - var codepoints = chars.Where(x => x != "" && x != "×") - .Select(x => Convert.ToInt32(x, 16)).ToArray(); - - var text = string.Join(null, codepoints.Select(char.ConvertFromUtf32)); - - var length = codepoints.Select(x => x > ushort.MaxValue ? 2 : 1).Sum(); - - var data = new object[] { text, length }; - - testData.Add(data); - } - } - } - } - - return testData; } } } diff --git a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/TestDataGenerator.cs b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/TestDataGenerator.cs new file mode 100644 index 0000000000..058de909df --- /dev/null +++ b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/TestDataGenerator.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; + +namespace Avalonia.Visuals.UnitTests.Media.TextFormatting +{ + public abstract class TestDataGenerator : IEnumerable + { + private readonly string _fileName; + private readonly List _testData; + + protected TestDataGenerator(string fileName) + { + _fileName = fileName; + _testData = ReadTestData(); + } + + public IEnumerator GetEnumerator() + { + return _testData.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + private List ReadTestData() + { + var testData = new List(); + + using (var client = new HttpClient()) + { + var url = Path.Combine(UnicodeDataGenerator.Ucd, _fileName); + + using (var result = client.GetAsync(url).GetAwaiter().GetResult()) + { + if (!result.IsSuccessStatusCode) + return testData; + + using (var stream = result.Content.ReadAsStreamAsync().GetAwaiter().GetResult()) + using (var reader = new StreamReader(stream)) + { + while (!reader.EndOfStream) + { + var line = reader.ReadLine(); + + if (line == null) + { + break; + } + + if (line.StartsWith("#") || string.IsNullOrEmpty(line)) + { + continue; + } + + var elements = line.Split('#'); + + elements = elements[0].Replace("÷\t", "÷").Trim('÷').Split('÷'); + + var chars = elements[0].Replace(" × ", " ").Split(' '); + + var codepoints = chars.Where(x => x != "" && x != "×") + .Select(x => Convert.ToInt32(x, 16)).ToArray(); + + var text = string.Join(null, codepoints.Select(char.ConvertFromUtf32)); + + var length = codepoints.Select(x => x > ushort.MaxValue ? 2 : 1).Sum(); + + var data = new object[] { text, length }; + + testData.Add(data); + } + } + } + } + + return testData; + } + } +} diff --git a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeDataGeneratorTests.cs b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeDataGeneratorTests.cs index c063cd296b..c79852e830 100644 --- a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeDataGeneratorTests.cs +++ b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeDataGeneratorTests.cs @@ -1,4 +1,6 @@ -using Xunit; +using System; +using Avalonia.Media.TextFormatting.Unicode; +using Xunit; namespace Avalonia.Visuals.UnitTests.Media.TextFormatting { @@ -13,5 +15,26 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting { UnicodeDataGenerator.Execute(); } + [Theory/*(Skip = "Only run when we update the trie.")*/] + [ClassData(typeof(LineBreakTestDataGenerator))] + + public void Should_Enumerate_LineBreaks(string text, int expectedLength) + { + var textMemory = text.AsMemory(); + + var enumerator = new LineBreakEnumerator(textMemory); + + Assert.True(enumerator.MoveNext()); + + Assert.Equal(expectedLength, enumerator.Current.PositionWrap); + } + + private class LineBreakTestDataGenerator : TestDataGenerator + { + public LineBreakTestDataGenerator() + : base("auxiliary/LineBreakTest.txt") + { + } + } } } diff --git a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeEnumsGenerator.cs b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeEnumsGenerator.cs index 413ebd645e..3a936ff3b0 100644 --- a/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeEnumsGenerator.cs +++ b/tests/Avalonia.Visuals.UnitTests/Media/TextFormatting/UnicodeEnumsGenerator.cs @@ -42,7 +42,7 @@ namespace Avalonia.Visuals.UnitTests.Media.TextFormatting public static List CreateGeneralCategoryEnum() { - var entries = new List { new DataEntry("Other", "C#", string.Empty) }; + var entries = new List { new DataEntry("Other", "C", " Cc | Cf | Cn | Co | Cs") }; ParseDataEntries("# General_Category (gc)", entries);