using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; using Avalonia.Media.TextFormatting.Unicode; using Xunit; namespace Avalonia.Base.UnitTests.Media.TextFormatting { public class WordBreakEnumeratorTests { private readonly ITestOutputHelper _outputHelper; public WordBreakEnumeratorTests(ITestOutputHelper outputHelper) { _outputHelper = outputHelper; } [Fact] public void ShouldReportCodepointReadouts() { var wordBreaker = new WordBreakEnumerator("hello world"); var segments = new List<(int Offset, int Length, int CodepointOffset, int CodepointLength, string Text)>(); while (wordBreaker.MoveNext(out var segment)) { segments.Add((segment.Offset, segment.Length, segment.CodepointOffset, segment.CodepointLength, segment.Text.ToString())); } Assert.Equal( new[] { (0, 5, 0, 5, "hello"), (5, 1, 5, 1, " "), (6, 5, 6, 5, "world") }, segments); } [Fact] public void ShouldKeepReadoutsConsistentWithText() { // Mixes a combining mark (WB4-absorbed), a surrogate pair and a CRLF so the // code-unit and code-point readouts diverge and every member is exercised. const string text = "áb \U0001D51Ex cr\r\nlf"; var wordBreaker = new WordBreakEnumerator(text); var offset = 0; var codepointOffset = 0; while (wordBreaker.MoveNext(out var segment)) { Assert.Equal(offset, segment.Offset); Assert.Equal(codepointOffset, segment.CodepointOffset); Assert.Equal(segment.Text.Length, segment.Length); Assert.True(text.AsSpan(segment.Offset, segment.Length).SequenceEqual(segment.Text)); var codepointCount = 0; var span = segment.Text; while (!span.IsEmpty) { Codepoint.ReadAt(span, 0, out var consumed); span = span.Slice(consumed); codepointCount++; } Assert.Equal(codepointCount, segment.CodepointLength); offset += segment.Length; codepointOffset += segment.CodepointLength; } Assert.Equal(text.Length, offset); } [Theory(Skip = "Only run when we update Unicode data.")] [ClassData(typeof(WordBreakTestDataGenerator))] public void ShouldFindBreaks(int lineNumber, int[] codePoints, int[] breakPoints, string rules) { var text = string.Join(null, codePoints.Select(char.ConvertFromUtf32)); var wordBreaker = new WordBreakEnumerator(text); var foundBreaks = new List { 0 }; var currentPosition = 0; while (wordBreaker.MoveNext(out var segment)) { currentPosition += segment.CodepointLength; foundBreaks.Add(currentPosition); } var pass = true; if (foundBreaks.Count != breakPoints.Length) { pass = false; } else { for (var i = 0; i < foundBreaks.Count; i++) { if (foundBreaks[i] != breakPoints[i]) { pass = false; } } } if (!pass) { _outputHelper.WriteLine($"Failed test on line {lineNumber}"); _outputHelper.WriteLine(""); _outputHelper.WriteLine($" Code Points: {string.Join(" ", codePoints)}"); _outputHelper.WriteLine($"Expected Breaks: {string.Join(" ", breakPoints)}"); _outputHelper.WriteLine($" Actual Breaks: {string.Join(" ", foundBreaks)}"); _outputHelper.WriteLine($" Text: {text}"); _outputHelper.WriteLine($" Char Props: {string.Join(" ", codePoints.Select(x => new Codepoint((uint)x).WordBreakClass))}"); _outputHelper.WriteLine($" Rules: {rules}"); _outputHelper.WriteLine(""); } Assert.True(pass); } // Regional indicators pair up from the start of their run (WB15, WB16), so where a // boundary falls depends on how many of them precede the current one. private const string RegionalD = "\U0001F1E9"; private const string RegionalE = "\U0001F1EA"; private const string RegionalF = "\U0001F1EB"; private const string RegionalR = "\U0001F1F7"; [Fact] public void TwoFlags_AreTwoSegments() { var segments = CollectSegments(RegionalD + RegionalE + RegionalF + RegionalR); Assert.Equal([RegionalD + RegionalE, RegionalF + RegionalR], segments); } [Fact] public void OddRegionalIndicatorRun_LeavesTheLastIndicatorOnItsOwn() { var segments = CollectSegments(RegionalD + RegionalE + RegionalF); Assert.Equal([RegionalD + RegionalE, RegionalF], segments); } [Fact] public void RegionalIndicatorsAfterALetter_PairFromTheStartOfTheirRun() { var segments = CollectSegments("A" + RegionalD + RegionalE + RegionalF + RegionalR); Assert.Equal(["A", RegionalD + RegionalE, RegionalF + RegionalR], segments); } [Fact] public void RegionalIndicatorRunInterruptedBySpace_StartsPairingAgain() { var segments = CollectSegments(RegionalD + " " + RegionalE + RegionalF); Assert.Equal([RegionalD, " ", RegionalE + RegionalF], segments); } [Fact] public void CombiningMarkBetweenRegionalIndicators_DoesNotSplitThePair() { var text = RegionalD + "\u0301" + RegionalE; var segments = CollectSegments(text); Assert.Equal([text], segments); } [Fact] public void LongRegionalIndicatorRun_PairsAllTheWayThrough() { var flag = RegionalD + RegionalE; var segments = CollectSegments(string.Concat(Enumerable.Repeat(flag, 8))); Assert.Equal(8, segments.Count); Assert.All(segments, segment => Assert.Equal(flag, segment)); } private static List CollectSegments(string text) { var result = new List(); var enumerator = new WordBreakEnumerator(text.AsSpan()); while (enumerator.MoveNext(out var segment)) { result.Add(text.Substring(segment.Offset, segment.Length)); } return result; } public class WordBreakTestDataGenerator : IEnumerable { private readonly List _testData; public WordBreakTestDataGenerator() { _testData = GenerateTestData(); } public IEnumerator GetEnumerator() { return _testData.GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } private static List GenerateTestData() { var tests = new List(); var url = Path.Combine(UnicodeDataSource.Ucd, "auxiliary/WordBreakTest.txt"); using (var client = new HttpClient()) using (var result = client.GetAsync(url).GetAwaiter().GetResult()) { if (!result.IsSuccessStatusCode) { return tests; } using (var stream = result.Content.ReadAsStreamAsync().GetAwaiter().GetResult()) using (var reader = new StreamReader(stream)) { var lineNumber = 1; while (!reader.EndOfStream) { var line = reader.ReadLine(); if (line is null) { break; } var segments = line.Split('#'); if (string.IsNullOrWhiteSpace(segments[0])) { lineNumber++; continue; } var lineData = ReadLineData(segments[0].Trim()); tests.Add([lineNumber, lineData.Item1, lineData.Item2, segments[1]]); lineNumber++; } } } return tests; } public static (int[], int[]) ReadLineData(string line) { var codePoints = new List(); var breakPoints = new List(); var p = 0; while (p < line.Length) { if (char.IsWhiteSpace(line[p])) { p++; continue; } if (line[p] == '×') { p++; continue; } if (line[p] == '÷') { breakPoints.Add(codePoints.Count); p++; continue; } var codePointPos = p; while (p < line.Length && IsHexDigit(line[p])) { p++; } var codePointStr = line.Substring(codePointPos, p - codePointPos); var codePoint = Convert.ToInt32(codePointStr, 16); codePoints.Add(codePoint); } return (codePoints.ToArray(), breakPoints.ToArray()); } private static bool IsHexDigit(char ch) { return char.IsDigit(ch) || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f'); } } } }