diff --git a/src/Data/Text/DelimitedReader.cs b/src/Data/Text/DelimitedReader.cs index d5f7b08c..7170b812 100644 --- a/src/Data/Text/DelimitedReader.cs +++ b/src/Data/Text/DelimitedReader.cs @@ -4,7 +4,7 @@ // http://github.com/mathnet/mathnet-numerics // http://mathnetnumerics.codeplex.com // -// Copyright (c) 2009-2014 Math.NET +// Copyright (c) 2009-2015 Math.NET // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation @@ -45,11 +45,16 @@ namespace MathNet.Numerics.Data.Text /// specify a delimiter, then any whitespace is used. /// public static class DelimitedReader - { + { /// /// The base regular expression. /// - const string RegexTemplate = "\\([^\\)]*\\)|'[^']*'|\"[^\"]*\"|[^{0}]*"; + const string RegexTemplate = "\\([^\\)]*\\)|'[^']*'|\"[^\"]*\"|^{0}{{1,}}|{0}$|{0}{{2,}}|[^{0}]*"; + + /// + /// Regular expression to match whitespace. + /// + const string WhiteSpaceRegexTemplate = "\\([^\\)]*\\)|'[^']*'|\"[^\"]*\"|[^\\s]*"; /// /// Cached compiled regular expressions for various delimiters, as needed. @@ -64,17 +69,19 @@ namespace MathNet.Numerics.Data.Text /// Number delimiter between numbers of the same line. Supports Regex groups. Default: "\s" (white space). /// Whether the first row contains column headers or not. Default: false. /// The culture to use. Default: null. + /// The value to represent missing values. Default: NaN. /// A matrix containing the data from the . /// The data type of the Matrix. It can be either: double, float, Complex, or Complex32. - public static Matrix Read(TextReader reader, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null) + public static Matrix Read(TextReader reader, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { - if (string.IsNullOrEmpty(delimiter)) - { - delimiter = @"\s"; - } - - var regex = RegexCache.GetOrAdd(delimiter, d => new Regex(string.Format(RegexTemplate, d), RegexOptions.Compiled)); + delimiter = CleanDelimiter(delimiter); + var mv = missingValue ?? NaN(); + var mvStr = mv.ToString(); + + var regex = delimiter == @"\s" ? + RegexCache.GetOrAdd(delimiter, d => new Regex(WhiteSpaceRegexTemplate, RegexOptions.Compiled)) : + RegexCache.GetOrAdd(delimiter, d => new Regex(string.Format(RegexTemplate, d), RegexOptions.Compiled)); var data = new List(); @@ -83,9 +90,9 @@ namespace MathNet.Numerics.Data.Text // 3,4,5,6 // 7 // this creates a 3x4 matrix: - // 1, 2, 0 ,0 + // 1, 2, MISSING, MISSING // 3, 4, 5, 6 - // 7, 0, 0, 0 + // 7, MISSING, MISSING, MISSING var max = -1; var line = reader.ReadLine(); @@ -97,19 +104,49 @@ namespace MathNet.Numerics.Data.Text while (line != null) { line = line.Trim(); + if (line.Length > 0) { var matches = regex.Matches(line); - var row = (from Match match in matches where match.Length > 0 select match.Value).ToArray(); - max = Math.Max(max, row.Length); - data.Add(row); - } + if (delimiter == @"\s") + { + var row = (from Match match in matches where match.Length > 0 select match.Value).ToArray(); + max = Math.Max(max, row.Length); + data.Add(row); + } + else + { + var offset = 0; + var row = new List(); + + foreach (var value in (from Match match in matches where match.Length > 0 select match.Value)) + { + var delimterCount = value.StartsWith(delimiter) ? value.CountDelimiters(delimiter) : 0; + + if (delimterCount > 0) + { + for (var i = offset; i < delimterCount; i++) + { + row.Add(mvStr); + } + } + else + { + row.Add(string.IsNullOrWhiteSpace(value) ? mvStr : value); + } + offset = 1; + } + + max = Math.Max(max, row.Count); + data.Add(row.ToArray()); + } + } line = reader.ReadLine(); } var parse = CreateParser(formatProvider); - var matrix = sparse ? Matrix.Build.Sparse(data.Count, max) : Matrix.Build.Dense(data.Count, max); + var matrix = sparse ? Matrix.Build.Sparse(data.Count, max, mv) : Matrix.Build.Dense(data.Count, max, mv); var storage = matrix.Storage; for (var i = 0; i < data.Count; i++) @@ -137,14 +174,15 @@ namespace MathNet.Numerics.Data.Text /// Number delimiter between numbers of the same line. Supports Regex groups. Default: "\s" (white space). /// Whether the first row contains column headers or not. Default: false. /// The culture to use. Default: null. + /// The value to represent missing values. Default: NaN. /// A matrix containing the data from the . /// The data type of the Matrix. It can be either: double, float, Complex, or Complex32. - public static Matrix Read(string filePath, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null) + public static Matrix Read(string filePath, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { using (var reader = new StreamReader(filePath)) { - return Read(reader, sparse, delimiter, hasHeaders, formatProvider); + return Read(reader, sparse, delimiter, hasHeaders, formatProvider, missingValue); } } @@ -156,15 +194,44 @@ namespace MathNet.Numerics.Data.Text /// Number delimiter between numbers of the same line. Supports Regex groups. Default: "\s" (white space). /// Whether the first row contains column headers or not. Default: false. /// The culture to use. Default: null. + /// The value to represent missing values. Default: NaN. /// A matrix containing the data from the . /// The data type of the Matrix. It can be either: double, float, Complex, or Complex32. - public static Matrix Read(Stream stream, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null) + public static Matrix Read(Stream stream, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null, T? missingValue=null) where T : struct, IEquatable, IFormattable { using (var reader = new StreamReader(stream)) { - return Read(reader, sparse, delimiter, hasHeaders, formatProvider); + return Read(reader, sparse, delimiter, hasHeaders, formatProvider, missingValue); + } + } + + /// + /// NaN for the given numeric type. + /// + private static T NaN() + { + if (typeof (T) == typeof (double)) + { + return (T)(object)double.NaN; + } + + if (typeof (T) == typeof (float)) + { + return (T)(object)float.NaN; + } + + if (typeof (T) == typeof (Complex)) + { + return (T)(object)new Complex(double.NaN, double.NaN); } + + if (typeof (T) == typeof (Complex32)) + { + return (T)(object)new Complex32(float.NaN, float.NaN); + } + + throw new NotSupportedException(); } static Func CreateParser(IFormatProvider formatProvider) @@ -191,5 +258,42 @@ namespace MathNet.Numerics.Data.Text throw new NotSupportedException(); } + + /// + /// Counts the number of delimiters in a row. + /// + private static int CountDelimiters(this string obj, string demlimiter) + { + var pos = 0; + var count = 0; + + while ((pos = obj.IndexOf(demlimiter, pos, StringComparison.InvariantCultureIgnoreCase)) != -1) + { + count++; + pos++; + } + + return count; + } + + /// + /// Returns a regex friendly delimter. + /// + private static string CleanDelimiter(string delimiter) + { + if (string.IsNullOrEmpty(delimiter)) + { + return @"\s"; + } + switch (delimiter) + { + case "." : + return "\\."; + case "\\" : + return "\\\\"; + default: + return delimiter; + } + } } } diff --git a/src/DataUnitTests/Text/DelimitedReaderTests.cs b/src/DataUnitTests/Text/DelimitedReaderTests.cs index 08483857..59dcad57 100644 --- a/src/DataUnitTests/Text/DelimitedReaderTests.cs +++ b/src/DataUnitTests/Text/DelimitedReaderTests.cs @@ -59,11 +59,11 @@ namespace MathNet.Numerics.Data.UnitTests.Text Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0, matrix[0, 0]); - Assert.AreEqual(0.0, matrix[0, 1]); - Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(double.NaN, matrix[0, 1]); + Assert.AreEqual(double.NaN, matrix[0, 2]); Assert.AreEqual(2.2, matrix[1, 0]); Assert.AreEqual(3.0, matrix[1, 1]); - Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(double.NaN, matrix[1, 2]); Assert.AreEqual(4.0, matrix[2, 0]); Assert.AreEqual(5.0, matrix[2, 1]); Assert.AreEqual(6.0, matrix[2, 2]); @@ -79,15 +79,15 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\"\t\t0.3e1" + Environment.NewLine + "'4'\t5\t6"; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: "\t", formatProvider: CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: "\t", formatProvider: CultureInfo.InvariantCulture, missingValue:0.0f); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0f, matrix[0, 0]); Assert.AreEqual(0.0f, matrix[0, 1]); Assert.AreEqual(0.0f, matrix[0, 2]); Assert.AreEqual(2.2f, matrix[1, 0]); - Assert.AreEqual(3.0f, matrix[1, 1]); - Assert.AreEqual(0.0f, matrix[1, 2]); + Assert.AreEqual(0.0f, matrix[1, 1]); + Assert.AreEqual(3.0f, matrix[1, 2]); Assert.AreEqual(4.0f, matrix[2, 0]); Assert.AreEqual(5.0f, matrix[2, 1]); Assert.AreEqual(6.0f, matrix[2, 2]); @@ -103,7 +103,7 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\" 0.3e1" + Environment.NewLine + "'4' 5 6" + Environment.NewLine; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), formatProvider: CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), formatProvider: CultureInfo.InvariantCulture, missingValue:0.0f); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0f, matrix[0, 0]); @@ -128,7 +128,7 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2,2\".0,3e1" + Environment.NewLine + "'4,0'.5,0.6,0" + Environment.NewLine; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ".", hasHeaders: true, formatProvider: new CultureInfo("tr-TR")); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ".", hasHeaders: true, formatProvider: new CultureInfo("tr-TR"), missingValue:0.0); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0, matrix[0, 0]); @@ -155,19 +155,20 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "'(4,-5)',5,6" + Environment.NewLine; var matrix = DelimitedReader.Read(new StringReader(data), delimiter: ",", hasHeaders: true, formatProvider: CultureInfo.InvariantCulture); + var nan = new Complex(double.NaN, double.NaN); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0, matrix[0, 0].Real); Assert.AreEqual(2.0, matrix[0, 0].Imaginary); - Assert.AreEqual((Complex) 0.0, matrix[0, 1]); - Assert.AreEqual((Complex) 0.0, matrix[0, 2]); - Assert.AreEqual((Complex) 2.2, matrix[1, 0]); - Assert.AreEqual((Complex) 3.0, matrix[1, 1]); - Assert.AreEqual((Complex) 0.0, matrix[1, 2]); + Assert.AreEqual(nan, matrix[0, 1]); + Assert.AreEqual(nan, matrix[0, 2]); + Assert.AreEqual((Complex)2.2, matrix[1, 0]); + Assert.AreEqual((Complex)3.0, matrix[1, 1]); + Assert.AreEqual(nan, matrix[1, 2]); Assert.AreEqual(4.0, matrix[2, 0].Real); Assert.AreEqual(-5.0, matrix[2, 0].Imaginary); - Assert.AreEqual((Complex) 5.0, matrix[2, 1]); - Assert.AreEqual((Complex) 6.0, matrix[2, 2]); + Assert.AreEqual((Complex)5.0, matrix[2, 1]); + Assert.AreEqual((Complex)6.0, matrix[2, 2]); } /// @@ -181,7 +182,7 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\",0.3e1" + Environment.NewLine + "'(4,-5)',5,6" + Environment.NewLine; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ",", hasHeaders: true, formatProvider: CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ",", hasHeaders: true, formatProvider: CultureInfo.InvariantCulture, missingValue:Complex32.Zero); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0f, matrix[0, 0].Real); @@ -208,7 +209,102 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\",0.3e1" + Environment.NewLine + "'4',0,6" + Environment.NewLine; - var matrix = DelimitedReader.Read(new StringReader(data), true, ",", true, CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new StringReader(data), true, ",", true, CultureInfo.InvariantCulture, missingValue:0.0); + Assert.IsTrue(matrix is LinearAlgebra.Double.SparseMatrix); + Assert.AreEqual(3, matrix.RowCount); + Assert.AreEqual(3, matrix.ColumnCount); + Assert.AreEqual(1.0, matrix[0, 0]); + Assert.AreEqual(0.0, matrix[0, 1]); + Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(3.0, matrix[1, 1]); + Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + } + + + /// + /// Can parse comma delimited data with missing data. + /// + [Test] + public void CanParseMissingDataCommaDelimitedData() + { + var data = "a,b,c" + Environment.NewLine + + ", , , ," + Environment.NewLine + + "\"2.2\", , ,0.3e1" + Environment.NewLine + + "'4',0,6" + Environment.NewLine + + "," + Environment.NewLine + + ",,3, 4" + Environment.NewLine + + ",, , ,," + Environment.NewLine + + ",,,,," + Environment.NewLine + + ", ,,, ," + Environment.NewLine + + ", , , ,," + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), false, ",", true, CultureInfo.InvariantCulture); + + Assert.AreEqual(9, matrix.RowCount); + Assert.AreEqual(5, matrix.ColumnCount); + Assert.AreEqual(double.NaN, matrix[0, 0]); + Assert.AreEqual(double.NaN, matrix[0, 1]); + Assert.AreEqual(double.NaN, matrix[0, 2]); + Assert.AreEqual(double.NaN, matrix[0, 3]); + Assert.AreEqual(double.NaN, matrix[0, 4]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(double.NaN, matrix[1, 1]); + Assert.AreEqual(double.NaN, matrix[1, 2]); + Assert.AreEqual(3.0, matrix[1, 3]); + Assert.AreEqual(double.NaN, matrix[1, 4]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + Assert.AreEqual(double.NaN, matrix[2, 3]); + Assert.AreEqual(double.NaN, matrix[2, 4]); + Assert.AreEqual(double.NaN, matrix[3, 0]); + Assert.AreEqual(double.NaN, matrix[3, 1]); + Assert.AreEqual(double.NaN, matrix[3, 2]); + Assert.AreEqual(double.NaN, matrix[3, 3]); + Assert.AreEqual(double.NaN, matrix[3, 4]); + Assert.AreEqual(double.NaN, matrix[4, 0]); + Assert.AreEqual(double.NaN, matrix[4, 1]); + Assert.AreEqual(3.0, matrix[4, 2]); + Assert.AreEqual(4.0, matrix[4, 3]); + Assert.AreEqual(double.NaN, matrix[4, 4]); + Assert.AreEqual(double.NaN, matrix[5, 0]); + Assert.AreEqual(double.NaN, matrix[5, 1]); + Assert.AreEqual(double.NaN, matrix[5, 2]); + Assert.AreEqual(double.NaN, matrix[5, 3]); + Assert.AreEqual(double.NaN, matrix[5, 4]); + Assert.AreEqual(double.NaN, matrix[6, 0]); + Assert.AreEqual(double.NaN, matrix[6, 1]); + Assert.AreEqual(double.NaN, matrix[6, 2]); + Assert.AreEqual(double.NaN, matrix[6, 3]); + Assert.AreEqual(double.NaN, matrix[6, 4]); + Assert.AreEqual(double.NaN, matrix[7, 0]); + Assert.AreEqual(double.NaN, matrix[7, 1]); + Assert.AreEqual(double.NaN, matrix[7, 2]); + Assert.AreEqual(double.NaN, matrix[7, 3]); + Assert.AreEqual(double.NaN, matrix[7, 4]); + Assert.AreEqual(double.NaN, matrix[8, 0]); + Assert.AreEqual(double.NaN, matrix[8, 1]); + Assert.AreEqual(double.NaN, matrix[8, 2]); + Assert.AreEqual(double.NaN, matrix[8, 3]); + Assert.AreEqual(double.NaN, matrix[8, 4]); + } + + /// + /// Can parse backslash delimited sparse data. + /// + [Test] + public void CanParseSparseBackSlashDelimitedData() + { + var data = "a\\b\\c" + Environment.NewLine + + "1" + Environment.NewLine + + "\"2.2\"\\0.3e1" + Environment.NewLine + + "'4'\\0\\6" + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), true, "\\", true, CultureInfo.InvariantCulture, missingValue:0.0); Assert.IsTrue(matrix is LinearAlgebra.Double.SparseMatrix); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); @@ -222,5 +318,56 @@ namespace MathNet.Numerics.Data.UnitTests.Text Assert.AreEqual(0.0, matrix[2, 1]); Assert.AreEqual(6.0, matrix[2, 2]); } + + /// + /// Can parse dash delimited sparse data. + /// + [Test] + public void CanParseSparseDashDelimitedData() + { + var data = "a-b-c" + Environment.NewLine + + "1" + Environment.NewLine + + "\"2.2\"-.3e1" + Environment.NewLine + + "'4'-0-6" + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), true, "-", true, CultureInfo.InvariantCulture, missingValue:0.0); + Assert.IsTrue(matrix is LinearAlgebra.Double.SparseMatrix); + Assert.AreEqual(3, matrix.RowCount); + Assert.AreEqual(3, matrix.ColumnCount); + Assert.AreEqual(1.0, matrix[0, 0]); + Assert.AreEqual(0.0, matrix[0, 1]); + Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(3.0, matrix[1, 1]); + Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + } + + /// + /// Can parse underscore delimited sparse data. + /// + [Test] + public void CanParseUnderscoreDelimitedData() + { + var data = "a_b_c" + Environment.NewLine + + "1" + Environment.NewLine + + "\"2.2\"_.3e1" + Environment.NewLine + + "'4'_0_6" + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), false, "_", true, CultureInfo.InvariantCulture, missingValue:0.0f); + Assert.AreEqual(3, matrix.RowCount); + Assert.AreEqual(3, matrix.ColumnCount); + Assert.AreEqual(1.0, matrix[0, 0]); + Assert.AreEqual(0.0, matrix[0, 1]); + Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(3.0, matrix[1, 1]); + Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + } } }