From 759975c7488e3a257f2c82b420d431fb4c50345b Mon Sep 17 00:00:00 2001 From: Marcus Cuda Date: Fri, 2 Jan 2015 14:25:07 +0200 Subject: [PATCH 1/2] added support for missing values --- src/Data/Text/DelimitedReader.cs | 146 ++++++++++++-- .../Text/DelimitedReaderTests.cs | 181 ++++++++++++++++-- 2 files changed, 289 insertions(+), 38 deletions(-) diff --git a/src/Data/Text/DelimitedReader.cs b/src/Data/Text/DelimitedReader.cs index d5f7b08c..7170b812 100644 --- a/src/Data/Text/DelimitedReader.cs +++ b/src/Data/Text/DelimitedReader.cs @@ -4,7 +4,7 @@ // http://github.com/mathnet/mathnet-numerics // http://mathnetnumerics.codeplex.com // -// Copyright (c) 2009-2014 Math.NET +// Copyright (c) 2009-2015 Math.NET // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation @@ -45,11 +45,16 @@ namespace MathNet.Numerics.Data.Text /// specify a delimiter, then any whitespace is used. /// public static class DelimitedReader - { + { /// /// The base regular expression. /// - const string RegexTemplate = "\\([^\\)]*\\)|'[^']*'|\"[^\"]*\"|[^{0}]*"; + const string RegexTemplate = "\\([^\\)]*\\)|'[^']*'|\"[^\"]*\"|^{0}{{1,}}|{0}$|{0}{{2,}}|[^{0}]*"; + + /// + /// Regular expression to match whitespace. + /// + const string WhiteSpaceRegexTemplate = "\\([^\\)]*\\)|'[^']*'|\"[^\"]*\"|[^\\s]*"; /// /// Cached compiled regular expressions for various delimiters, as needed. @@ -64,17 +69,19 @@ namespace MathNet.Numerics.Data.Text /// Number delimiter between numbers of the same line. Supports Regex groups. Default: "\s" (white space). /// Whether the first row contains column headers or not. Default: false. /// The culture to use. Default: null. + /// The value to represent missing values. Default: NaN. /// A matrix containing the data from the . /// The data type of the Matrix. It can be either: double, float, Complex, or Complex32. - public static Matrix Read(TextReader reader, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null) + public static Matrix Read(TextReader reader, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { - if (string.IsNullOrEmpty(delimiter)) - { - delimiter = @"\s"; - } - - var regex = RegexCache.GetOrAdd(delimiter, d => new Regex(string.Format(RegexTemplate, d), RegexOptions.Compiled)); + delimiter = CleanDelimiter(delimiter); + var mv = missingValue ?? NaN(); + var mvStr = mv.ToString(); + + var regex = delimiter == @"\s" ? + RegexCache.GetOrAdd(delimiter, d => new Regex(WhiteSpaceRegexTemplate, RegexOptions.Compiled)) : + RegexCache.GetOrAdd(delimiter, d => new Regex(string.Format(RegexTemplate, d), RegexOptions.Compiled)); var data = new List(); @@ -83,9 +90,9 @@ namespace MathNet.Numerics.Data.Text // 3,4,5,6 // 7 // this creates a 3x4 matrix: - // 1, 2, 0 ,0 + // 1, 2, MISSING, MISSING // 3, 4, 5, 6 - // 7, 0, 0, 0 + // 7, MISSING, MISSING, MISSING var max = -1; var line = reader.ReadLine(); @@ -97,19 +104,49 @@ namespace MathNet.Numerics.Data.Text while (line != null) { line = line.Trim(); + if (line.Length > 0) { var matches = regex.Matches(line); - var row = (from Match match in matches where match.Length > 0 select match.Value).ToArray(); - max = Math.Max(max, row.Length); - data.Add(row); - } + if (delimiter == @"\s") + { + var row = (from Match match in matches where match.Length > 0 select match.Value).ToArray(); + max = Math.Max(max, row.Length); + data.Add(row); + } + else + { + var offset = 0; + var row = new List(); + + foreach (var value in (from Match match in matches where match.Length > 0 select match.Value)) + { + var delimterCount = value.StartsWith(delimiter) ? value.CountDelimiters(delimiter) : 0; + + if (delimterCount > 0) + { + for (var i = offset; i < delimterCount; i++) + { + row.Add(mvStr); + } + } + else + { + row.Add(string.IsNullOrWhiteSpace(value) ? mvStr : value); + } + offset = 1; + } + + max = Math.Max(max, row.Count); + data.Add(row.ToArray()); + } + } line = reader.ReadLine(); } var parse = CreateParser(formatProvider); - var matrix = sparse ? Matrix.Build.Sparse(data.Count, max) : Matrix.Build.Dense(data.Count, max); + var matrix = sparse ? Matrix.Build.Sparse(data.Count, max, mv) : Matrix.Build.Dense(data.Count, max, mv); var storage = matrix.Storage; for (var i = 0; i < data.Count; i++) @@ -137,14 +174,15 @@ namespace MathNet.Numerics.Data.Text /// Number delimiter between numbers of the same line. Supports Regex groups. Default: "\s" (white space). /// Whether the first row contains column headers or not. Default: false. /// The culture to use. Default: null. + /// The value to represent missing values. Default: NaN. /// A matrix containing the data from the . /// The data type of the Matrix. It can be either: double, float, Complex, or Complex32. - public static Matrix Read(string filePath, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null) + public static Matrix Read(string filePath, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { using (var reader = new StreamReader(filePath)) { - return Read(reader, sparse, delimiter, hasHeaders, formatProvider); + return Read(reader, sparse, delimiter, hasHeaders, formatProvider, missingValue); } } @@ -156,15 +194,44 @@ namespace MathNet.Numerics.Data.Text /// Number delimiter between numbers of the same line. Supports Regex groups. Default: "\s" (white space). /// Whether the first row contains column headers or not. Default: false. /// The culture to use. Default: null. + /// The value to represent missing values. Default: NaN. /// A matrix containing the data from the . /// The data type of the Matrix. It can be either: double, float, Complex, or Complex32. - public static Matrix Read(Stream stream, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null) + public static Matrix Read(Stream stream, bool sparse = false, string delimiter = @"\s", bool hasHeaders = false, IFormatProvider formatProvider = null, T? missingValue=null) where T : struct, IEquatable, IFormattable { using (var reader = new StreamReader(stream)) { - return Read(reader, sparse, delimiter, hasHeaders, formatProvider); + return Read(reader, sparse, delimiter, hasHeaders, formatProvider, missingValue); + } + } + + /// + /// NaN for the given numeric type. + /// + private static T NaN() + { + if (typeof (T) == typeof (double)) + { + return (T)(object)double.NaN; + } + + if (typeof (T) == typeof (float)) + { + return (T)(object)float.NaN; + } + + if (typeof (T) == typeof (Complex)) + { + return (T)(object)new Complex(double.NaN, double.NaN); } + + if (typeof (T) == typeof (Complex32)) + { + return (T)(object)new Complex32(float.NaN, float.NaN); + } + + throw new NotSupportedException(); } static Func CreateParser(IFormatProvider formatProvider) @@ -191,5 +258,42 @@ namespace MathNet.Numerics.Data.Text throw new NotSupportedException(); } + + /// + /// Counts the number of delimiters in a row. + /// + private static int CountDelimiters(this string obj, string demlimiter) + { + var pos = 0; + var count = 0; + + while ((pos = obj.IndexOf(demlimiter, pos, StringComparison.InvariantCultureIgnoreCase)) != -1) + { + count++; + pos++; + } + + return count; + } + + /// + /// Returns a regex friendly delimter. + /// + private static string CleanDelimiter(string delimiter) + { + if (string.IsNullOrEmpty(delimiter)) + { + return @"\s"; + } + switch (delimiter) + { + case "." : + return "\\."; + case "\\" : + return "\\\\"; + default: + return delimiter; + } + } } } diff --git a/src/DataUnitTests/Text/DelimitedReaderTests.cs b/src/DataUnitTests/Text/DelimitedReaderTests.cs index 08483857..59dcad57 100644 --- a/src/DataUnitTests/Text/DelimitedReaderTests.cs +++ b/src/DataUnitTests/Text/DelimitedReaderTests.cs @@ -59,11 +59,11 @@ namespace MathNet.Numerics.Data.UnitTests.Text Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0, matrix[0, 0]); - Assert.AreEqual(0.0, matrix[0, 1]); - Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(double.NaN, matrix[0, 1]); + Assert.AreEqual(double.NaN, matrix[0, 2]); Assert.AreEqual(2.2, matrix[1, 0]); Assert.AreEqual(3.0, matrix[1, 1]); - Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(double.NaN, matrix[1, 2]); Assert.AreEqual(4.0, matrix[2, 0]); Assert.AreEqual(5.0, matrix[2, 1]); Assert.AreEqual(6.0, matrix[2, 2]); @@ -79,15 +79,15 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\"\t\t0.3e1" + Environment.NewLine + "'4'\t5\t6"; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: "\t", formatProvider: CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: "\t", formatProvider: CultureInfo.InvariantCulture, missingValue:0.0f); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0f, matrix[0, 0]); Assert.AreEqual(0.0f, matrix[0, 1]); Assert.AreEqual(0.0f, matrix[0, 2]); Assert.AreEqual(2.2f, matrix[1, 0]); - Assert.AreEqual(3.0f, matrix[1, 1]); - Assert.AreEqual(0.0f, matrix[1, 2]); + Assert.AreEqual(0.0f, matrix[1, 1]); + Assert.AreEqual(3.0f, matrix[1, 2]); Assert.AreEqual(4.0f, matrix[2, 0]); Assert.AreEqual(5.0f, matrix[2, 1]); Assert.AreEqual(6.0f, matrix[2, 2]); @@ -103,7 +103,7 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\" 0.3e1" + Environment.NewLine + "'4' 5 6" + Environment.NewLine; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), formatProvider: CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), formatProvider: CultureInfo.InvariantCulture, missingValue:0.0f); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0f, matrix[0, 0]); @@ -128,7 +128,7 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2,2\".0,3e1" + Environment.NewLine + "'4,0'.5,0.6,0" + Environment.NewLine; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ".", hasHeaders: true, formatProvider: new CultureInfo("tr-TR")); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ".", hasHeaders: true, formatProvider: new CultureInfo("tr-TR"), missingValue:0.0); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0, matrix[0, 0]); @@ -155,19 +155,20 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "'(4,-5)',5,6" + Environment.NewLine; var matrix = DelimitedReader.Read(new StringReader(data), delimiter: ",", hasHeaders: true, formatProvider: CultureInfo.InvariantCulture); + var nan = new Complex(double.NaN, double.NaN); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0, matrix[0, 0].Real); Assert.AreEqual(2.0, matrix[0, 0].Imaginary); - Assert.AreEqual((Complex) 0.0, matrix[0, 1]); - Assert.AreEqual((Complex) 0.0, matrix[0, 2]); - Assert.AreEqual((Complex) 2.2, matrix[1, 0]); - Assert.AreEqual((Complex) 3.0, matrix[1, 1]); - Assert.AreEqual((Complex) 0.0, matrix[1, 2]); + Assert.AreEqual(nan, matrix[0, 1]); + Assert.AreEqual(nan, matrix[0, 2]); + Assert.AreEqual((Complex)2.2, matrix[1, 0]); + Assert.AreEqual((Complex)3.0, matrix[1, 1]); + Assert.AreEqual(nan, matrix[1, 2]); Assert.AreEqual(4.0, matrix[2, 0].Real); Assert.AreEqual(-5.0, matrix[2, 0].Imaginary); - Assert.AreEqual((Complex) 5.0, matrix[2, 1]); - Assert.AreEqual((Complex) 6.0, matrix[2, 2]); + Assert.AreEqual((Complex)5.0, matrix[2, 1]); + Assert.AreEqual((Complex)6.0, matrix[2, 2]); } /// @@ -181,7 +182,7 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\",0.3e1" + Environment.NewLine + "'(4,-5)',5,6" + Environment.NewLine; - var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ",", hasHeaders: true, formatProvider: CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new MemoryStream(Encoding.UTF8.GetBytes(data)), delimiter: ",", hasHeaders: true, formatProvider: CultureInfo.InvariantCulture, missingValue:Complex32.Zero); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); Assert.AreEqual(1.0f, matrix[0, 0].Real); @@ -208,7 +209,102 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "\"2.2\",0.3e1" + Environment.NewLine + "'4',0,6" + Environment.NewLine; - var matrix = DelimitedReader.Read(new StringReader(data), true, ",", true, CultureInfo.InvariantCulture); + var matrix = DelimitedReader.Read(new StringReader(data), true, ",", true, CultureInfo.InvariantCulture, missingValue:0.0); + Assert.IsTrue(matrix is LinearAlgebra.Double.SparseMatrix); + Assert.AreEqual(3, matrix.RowCount); + Assert.AreEqual(3, matrix.ColumnCount); + Assert.AreEqual(1.0, matrix[0, 0]); + Assert.AreEqual(0.0, matrix[0, 1]); + Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(3.0, matrix[1, 1]); + Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + } + + + /// + /// Can parse comma delimited data with missing data. + /// + [Test] + public void CanParseMissingDataCommaDelimitedData() + { + var data = "a,b,c" + Environment.NewLine + + ", , , ," + Environment.NewLine + + "\"2.2\", , ,0.3e1" + Environment.NewLine + + "'4',0,6" + Environment.NewLine + + "," + Environment.NewLine + + ",,3, 4" + Environment.NewLine + + ",, , ,," + Environment.NewLine + + ",,,,," + Environment.NewLine + + ", ,,, ," + Environment.NewLine + + ", , , ,," + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), false, ",", true, CultureInfo.InvariantCulture); + + Assert.AreEqual(9, matrix.RowCount); + Assert.AreEqual(5, matrix.ColumnCount); + Assert.AreEqual(double.NaN, matrix[0, 0]); + Assert.AreEqual(double.NaN, matrix[0, 1]); + Assert.AreEqual(double.NaN, matrix[0, 2]); + Assert.AreEqual(double.NaN, matrix[0, 3]); + Assert.AreEqual(double.NaN, matrix[0, 4]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(double.NaN, matrix[1, 1]); + Assert.AreEqual(double.NaN, matrix[1, 2]); + Assert.AreEqual(3.0, matrix[1, 3]); + Assert.AreEqual(double.NaN, matrix[1, 4]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + Assert.AreEqual(double.NaN, matrix[2, 3]); + Assert.AreEqual(double.NaN, matrix[2, 4]); + Assert.AreEqual(double.NaN, matrix[3, 0]); + Assert.AreEqual(double.NaN, matrix[3, 1]); + Assert.AreEqual(double.NaN, matrix[3, 2]); + Assert.AreEqual(double.NaN, matrix[3, 3]); + Assert.AreEqual(double.NaN, matrix[3, 4]); + Assert.AreEqual(double.NaN, matrix[4, 0]); + Assert.AreEqual(double.NaN, matrix[4, 1]); + Assert.AreEqual(3.0, matrix[4, 2]); + Assert.AreEqual(4.0, matrix[4, 3]); + Assert.AreEqual(double.NaN, matrix[4, 4]); + Assert.AreEqual(double.NaN, matrix[5, 0]); + Assert.AreEqual(double.NaN, matrix[5, 1]); + Assert.AreEqual(double.NaN, matrix[5, 2]); + Assert.AreEqual(double.NaN, matrix[5, 3]); + Assert.AreEqual(double.NaN, matrix[5, 4]); + Assert.AreEqual(double.NaN, matrix[6, 0]); + Assert.AreEqual(double.NaN, matrix[6, 1]); + Assert.AreEqual(double.NaN, matrix[6, 2]); + Assert.AreEqual(double.NaN, matrix[6, 3]); + Assert.AreEqual(double.NaN, matrix[6, 4]); + Assert.AreEqual(double.NaN, matrix[7, 0]); + Assert.AreEqual(double.NaN, matrix[7, 1]); + Assert.AreEqual(double.NaN, matrix[7, 2]); + Assert.AreEqual(double.NaN, matrix[7, 3]); + Assert.AreEqual(double.NaN, matrix[7, 4]); + Assert.AreEqual(double.NaN, matrix[8, 0]); + Assert.AreEqual(double.NaN, matrix[8, 1]); + Assert.AreEqual(double.NaN, matrix[8, 2]); + Assert.AreEqual(double.NaN, matrix[8, 3]); + Assert.AreEqual(double.NaN, matrix[8, 4]); + } + + /// + /// Can parse backslash delimited sparse data. + /// + [Test] + public void CanParseSparseBackSlashDelimitedData() + { + var data = "a\\b\\c" + Environment.NewLine + + "1" + Environment.NewLine + + "\"2.2\"\\0.3e1" + Environment.NewLine + + "'4'\\0\\6" + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), true, "\\", true, CultureInfo.InvariantCulture, missingValue:0.0); Assert.IsTrue(matrix is LinearAlgebra.Double.SparseMatrix); Assert.AreEqual(3, matrix.RowCount); Assert.AreEqual(3, matrix.ColumnCount); @@ -222,5 +318,56 @@ namespace MathNet.Numerics.Data.UnitTests.Text Assert.AreEqual(0.0, matrix[2, 1]); Assert.AreEqual(6.0, matrix[2, 2]); } + + /// + /// Can parse dash delimited sparse data. + /// + [Test] + public void CanParseSparseDashDelimitedData() + { + var data = "a-b-c" + Environment.NewLine + + "1" + Environment.NewLine + + "\"2.2\"-.3e1" + Environment.NewLine + + "'4'-0-6" + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), true, "-", true, CultureInfo.InvariantCulture, missingValue:0.0); + Assert.IsTrue(matrix is LinearAlgebra.Double.SparseMatrix); + Assert.AreEqual(3, matrix.RowCount); + Assert.AreEqual(3, matrix.ColumnCount); + Assert.AreEqual(1.0, matrix[0, 0]); + Assert.AreEqual(0.0, matrix[0, 1]); + Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(3.0, matrix[1, 1]); + Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + } + + /// + /// Can parse underscore delimited sparse data. + /// + [Test] + public void CanParseUnderscoreDelimitedData() + { + var data = "a_b_c" + Environment.NewLine + + "1" + Environment.NewLine + + "\"2.2\"_.3e1" + Environment.NewLine + + "'4'_0_6" + Environment.NewLine; + + var matrix = DelimitedReader.Read(new StringReader(data), false, "_", true, CultureInfo.InvariantCulture, missingValue:0.0f); + Assert.AreEqual(3, matrix.RowCount); + Assert.AreEqual(3, matrix.ColumnCount); + Assert.AreEqual(1.0, matrix[0, 0]); + Assert.AreEqual(0.0, matrix[0, 1]); + Assert.AreEqual(0.0, matrix[0, 2]); + Assert.AreEqual(2.2, matrix[1, 0]); + Assert.AreEqual(3.0, matrix[1, 1]); + Assert.AreEqual(0.0, matrix[1, 2]); + Assert.AreEqual(4.0, matrix[2, 0]); + Assert.AreEqual(0.0, matrix[2, 1]); + Assert.AreEqual(6.0, matrix[2, 2]); + } } } From f5ad18f239471eda5c3b4412d1be52e380493378 Mon Sep 17 00:00:00 2001 From: Marcus Cuda Date: Sat, 10 Jan 2015 13:15:19 +0200 Subject: [PATCH 2/2] added missing value support to DelimitedWriter --- src/Data/Text/DelimitedWriter.cs | 61 +++++++++++++++---- .../Text/DelimitedWriterTests.cs | 37 +++++++++++ 2 files changed, 85 insertions(+), 13 deletions(-) diff --git a/src/Data/Text/DelimitedWriter.cs b/src/Data/Text/DelimitedWriter.cs index 42e09012..198cb52a 100644 --- a/src/Data/Text/DelimitedWriter.cs +++ b/src/Data/Text/DelimitedWriter.cs @@ -50,9 +50,11 @@ namespace MathNet.Numerics.Data.Text /// Custom column header. Headers are only written if non-null and non-empty headers are provided. Default: null. /// The number format to use on each element. Default: null. /// The culture to use. Default: null. + /// A value that represents a missing value. If not null, then elemements of the matrix that have this value + /// are not written to the output file. /// If either or is null. /// The data type of the Matrix. It can be either: Double, Single, Complex, or Complex32. - public static void Write(TextWriter writer, Matrix matrix, string delimiter = "\t", IList columnHeaders = null, string format = null, IFormatProvider formatProvider = null) + public static void Write(TextWriter writer, Matrix matrix, string delimiter = "\t", IList columnHeaders = null, string format = null, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { if (matrix == null) @@ -78,20 +80,49 @@ namespace MathNet.Numerics.Data.Text var cols = matrix.ColumnCount - 1; var rows = matrix.RowCount - 1; - for (var i = 0; i < matrix.RowCount; i++) + if (!missingValue.HasValue) { - for (var j = 0; j < matrix.ColumnCount; j++) + for (var i = 0; i < matrix.RowCount; i++) { - writer.Write(matrix[i, j].ToString(format, formatProvider)); - if (j != cols) + for (var j = 0; j < matrix.ColumnCount; j++) { - writer.Write(delimiter); + writer.Write(matrix[i, j].ToString(format, formatProvider)); + + if (j != cols) + { + writer.Write(delimiter); + } } - } - if (i != rows) + if (i != rows) + { + writer.Write(Environment.NewLine); + } + } + } + else + { + var missing = missingValue.Value; + + for (var i = 0; i < matrix.RowCount; i++) { - writer.Write(Environment.NewLine); + for (var j = 0; j < matrix.ColumnCount; j++) + { + if (!matrix[i, j].Equals(missing)) + { + writer.Write(matrix[i, j].ToString(format, formatProvider)); + } + + if (j != cols) + { + writer.Write(delimiter); + } + } + + if (i != rows) + { + writer.Write(Environment.NewLine); + } } } } @@ -105,14 +136,16 @@ namespace MathNet.Numerics.Data.Text /// Custom column header. Headers are only written if non-null and non-empty headers are provided. Default: null. /// The number format to use on each element. Default: null. /// The culture to use. Default: null. + /// A value that represents a missing value. If not null, then elemements of the matrix that have this value + /// are not written to the output file. /// If either or is null. /// The data type of the Matrix. It can be either: Double, Single, Complex, or Complex32. - public static void Write(string filePath, Matrix matrix, string delimiter = "\t", IList columnHeaders = null, string format = null, IFormatProvider formatProvider = null) + public static void Write(string filePath, Matrix matrix, string delimiter = "\t", IList columnHeaders = null, string format = null, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { using (var writer = new StreamWriter(filePath)) { - Write(writer, matrix, delimiter: delimiter, columnHeaders: columnHeaders, format: format, formatProvider: formatProvider); + Write(writer, matrix, delimiter: delimiter, columnHeaders: columnHeaders, format: format, formatProvider: formatProvider, missingValue: missingValue); } } @@ -125,14 +158,16 @@ namespace MathNet.Numerics.Data.Text /// Custom column header. Headers are only written if non-null and non-empty headers are provided. Default: null. /// The number format to use on each element. Default: null. /// The culture to use. Default: null. + /// A value that represents a missing value. If not null, then elemements of the matrix that have this value + /// are not written to the output file. /// If either or is null. /// The data type of the Matrix. It can be either: Double, Single, Complex, or Complex32. - public static void Write(Stream stream, Matrix matrix, string delimiter = "\t", IList columnHeaders = null, string format = null, IFormatProvider formatProvider = null) + public static void Write(Stream stream, Matrix matrix, string delimiter = "\t", IList columnHeaders = null, string format = null, IFormatProvider formatProvider = null, T? missingValue = null) where T : struct, IEquatable, IFormattable { using (var writer = new StreamWriter(stream)) { - Write(writer, matrix, delimiter: delimiter, columnHeaders: columnHeaders, format: format, formatProvider: formatProvider); + Write(writer, matrix, delimiter: delimiter, columnHeaders: columnHeaders, format: format, formatProvider: formatProvider, missingValue: missingValue); } } } diff --git a/src/DataUnitTests/Text/DelimitedWriterTests.cs b/src/DataUnitTests/Text/DelimitedWriterTests.cs index bc364ff3..604d32a4 100644 --- a/src/DataUnitTests/Text/DelimitedWriterTests.cs +++ b/src/DataUnitTests/Text/DelimitedWriterTests.cs @@ -151,5 +151,42 @@ namespace MathNet.Numerics.Data.UnitTests.Text + "0 0 9.9"; Assert.AreEqual(expected, text); } + + /// + /// Can write comma delimited data with missing values. + /// + [Test] + public void CanWriteCommaDelimitedDataWithMissingValues() + { + var matrix = SparseMatrix.OfArray(new[,] { { 1.1, 0, 0 }, { 0, 5.5, 0 }, { 0, 0, 9.9 } }); + var stream = new MemoryStream(); + DelimitedWriter.Write(stream, matrix, ",", missingValue: 0); + var data = stream.ToArray(); + var reader = new StreamReader(new MemoryStream(data)); + var text = reader.ReadToEnd(); + var expected = @"1.1,," + Environment.NewLine + + ",5.5," + Environment.NewLine + + ",,9.9"; + Assert.AreEqual(expected, text); + } + + + /// + /// Can write space delimited data with missing values. + /// + [Test] + public void CanWriteTabDelimitedDataWithMissingValues() + { + var matrix = DenseMatrix.OfArray(new[,] { { 1.1, Double.NaN, 0 }, { 0, 5.5, 0 }, { Double.NaN, Double.NaN, 9.9 } }); + var stream = new MemoryStream(); + DelimitedWriter.Write(stream, matrix, "\t", missingValue: Double.NaN); + var data = stream.ToArray(); + var reader = new StreamReader(new MemoryStream(data)); + var text = reader.ReadToEnd(); + var expected = "1.1\t\t0" + Environment.NewLine + + "0\t5.5\t0" + Environment.NewLine + + "\t\t9.9"; + Assert.AreEqual(expected, text); + } } } \ No newline at end of file