Browse Source

Statistics: introduce StreamingStatistics, always throw on null data

v2
Christoph Ruegg 14 years ago
parent
commit
505625c172
  1. 1
      src/Numerics/Numerics.csproj
  2. 11
      src/Numerics/Statistics/ArrayStatistics.cs
  3. 17
      src/Numerics/Statistics/SortedArrayStatistics.cs
  4. 82
      src/Numerics/Statistics/StreamingStatistics.cs
  5. 3
      src/Portable/Portable.csproj
  6. 130
      src/UnitTests/StatisticsTests/StatisticsTests.cs

1
src/Numerics/Numerics.csproj

@ -112,6 +112,7 @@
<Compile Include="SpecialFunctions\ModifiedBessel.cs" />
<Compile Include="SpecialFunctions\Logistic.cs" />
<Compile Include="Statistics\ArrayStatistics.cs" />
<Compile Include="Statistics\StreamingStatistics.cs" />
<Compile Include="Statistics\SortedArrayStatistics.cs" />
<Compile Include="TargetedPatchingOptOutAttribute.cs" />
<Compile Include="Distributions\Continuous\Cauchy.cs" />

11
src/Numerics/Statistics/ArrayStatistics.cs

@ -28,6 +28,8 @@
// OTHER DEALINGS IN THE SOFTWARE.
// </copyright>
using System;
namespace MathNet.Numerics.Statistics
{
public static class ArrayStatistics
@ -42,7 +44,8 @@ namespace MathNet.Numerics.Statistics
/// <param name="data">Sample array, no sorting is assumed.</param>
public static double Minimum(double[] data)
{
if (data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (data.Length == 0) return double.NaN;
var min = double.PositiveInfinity;
for (int i = 0; i < data.Length; i++)
@ -62,7 +65,8 @@ namespace MathNet.Numerics.Statistics
/// <param name="data">Sample array, no sorting is assumed.</param>
public static double Maximum(double[] data)
{
if (data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (data.Length == 0) return double.NaN;
var max = double.NegativeInfinity;
for (int i = 0; i < data.Length; i++)
@ -82,7 +86,8 @@ namespace MathNet.Numerics.Statistics
/// <param name="data">Sample array, no sorting is assumed.</param>
public static double Mean(double[] data)
{
if (data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (data.Length == 0) return double.NaN;
double mean = 0;
ulong m = 0;

17
src/Numerics/Statistics/SortedArrayStatistics.cs

@ -51,7 +51,9 @@ namespace MathNet.Numerics.Statistics
/// <param name="data">Sample array, must be sorted ascendingly.</param>
public static double Minimum(double[] data)
{
if (data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (data.Length == 0) return double.NaN;
return data[0];
}
@ -61,7 +63,9 @@ namespace MathNet.Numerics.Statistics
/// <param name="data">Sample array, must be sorted ascendingly.</param>
public static double Maximum(double[] data)
{
if (data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (data.Length == 0) return double.NaN;
return data[data.Length - 1];
}
@ -124,7 +128,8 @@ namespace MathNet.Numerics.Statistics
/// <param name="data">Sample array, must be sorted ascendingly.</param>
public static double[] FiveNumberSummary(double[] data)
{
if (data == null || data.Length == 0) return new[] {double.NaN, double.NaN, double.NaN, double.NaN, double.NaN};
if (data == null) throw new ArgumentNullException("data");
if (data.Length == 0) return new[] {double.NaN, double.NaN, double.NaN, double.NaN, double.NaN};
return new[] {data[0], Quantile(data, 0.25), Quantile(data, 0.50), Quantile(data, 0.75), data[data.Length - 1]};
}
@ -142,7 +147,8 @@ namespace MathNet.Numerics.Statistics
/// </remarks>
public static double Quantile(double[] data, double tau)
{
if (tau < 0d || tau > 1d || data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (tau < 0d || tau > 1d || data.Length == 0) return double.NaN;
if (tau == 0d || data.Length == 1) return data[0];
if (tau == 1d) return data[data.Length - 1];
@ -158,7 +164,8 @@ namespace MathNet.Numerics.Statistics
/// </summary>
public static double QuantileCompatible(double[] data, double tau, QuantileCompatibility compatibility)
{
if (tau < 0d || tau > 1d || data == null || data.Length == 0) return double.NaN;
if (data == null) throw new ArgumentNullException("data");
if (tau < 0d || tau > 1d || data.Length == 0) return double.NaN;
if (tau == 0d || data.Length == 1) return data[0];
if (tau == 1d) return data[data.Length - 1];

82
src/Numerics/Statistics/StreamingStatistics.cs

@ -0,0 +1,82 @@
// <copyright file="StreamingStatistics.cs" company="Math.NET">
// Math.NET Numerics, part of the Math.NET Project
// http://numerics.mathdotnet.com
// http://github.com/mathnet/mathnet-numerics
// http://mathnetnumerics.codeplex.com
//
// Copyright (c) 2009-2013 Math.NET
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
// </copyright>
using System;
using System.Collections.Generic;
namespace MathNet.Numerics.Statistics
{
public static class StreamingStatistics
{
/// <summary>
/// Returns the smallest value from the enumerable, in a single pass without memoization.
/// Returns NaN if data is empty or any entry is NaN.
/// </summary>
/// <param name="stream">Sample stream, no sorting is assumed.</param>
public static double Minimum(IEnumerable<double> stream)
{
if (stream == null) throw new ArgumentNullException("stream");
var min = double.PositiveInfinity;
bool any = false;
foreach (double d in stream)
{
if (d < min || double.IsNaN(d))
{
min = d;
}
any = true;
}
return any ? min : double.NaN;
}
/// <summary>
/// Returns the largest value from the enumerable, in a single pass without memoization.
/// Returns NaN if data is empty or any entry is NaN.
/// </summary>
/// <param name="stream">Sample stream, no sorting is assumed.</param>
public static double Maximum(IEnumerable<double> stream)
{
if (stream == null) throw new ArgumentNullException("stream");
var max = double.NegativeInfinity;
bool any = false;
foreach (double d in stream)
{
if (d > max || double.IsNaN(d))
{
max = d;
}
any = true;
}
return any ? max : double.NaN;
}
}
}

3
src/Portable/Portable.csproj

@ -1068,6 +1068,9 @@
<Compile Include="..\Numerics\Statistics\Statistics.cs">
<Link>Statistics\Statistics.cs</Link>
</Compile>
<Compile Include="..\Numerics\Statistics\StreamingStatistics.cs">
<Link>Statistics\StreamingStatistics.cs</Link>
</Compile>
<Compile Include="..\Numerics\TargetedPatchingOptOutAttribute.cs">
<Link>TargetedPatchingOptOutAttribute.cs</Link>
</Compile>

130
src/UnitTests/StatisticsTests/StatisticsTests.cs

@ -30,7 +30,6 @@
namespace MathNet.Numerics.UnitTests.StatisticsTests
{
#if !PORTABLE
using System;
using System.Collections.Generic;
using System.IO;
@ -39,45 +38,79 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
using Statistics;
/// <summary>
/// Statistics tests.
/// Statistics Tests.
/// </summary>
/// <remarks>NOTE: this class is not included into Silverlight version, because it uses data from local files.
/// In Silverlight access to local files is forbidden, except several cases.</remarks>
[TestFixture]
public class StatisticsTests
{
/// <summary>
/// Statistics data.
/// </summary>
readonly IDictionary<string, StatTestData> _data = new Dictionary<string, StatTestData>();
readonly IDictionary<string, StatTestData> _data = new Dictionary<string, StatTestData>
{
{"lottery", new StatTestData("./data/NIST/Lottery.dat")},
{"lew", new StatTestData("./data/NIST/Lew.dat")},
{"mavro", new StatTestData("./data/NIST/Mavro.dat")},
{"michelso", new StatTestData("./data/NIST/Michelso.dat")},
{"numacc1", new StatTestData("./data/NIST/NumAcc1.dat")},
{"numacc2", new StatTestData("./data/NIST/NumAcc2.dat")},
{"numacc3", new StatTestData("./data/NIST/NumAcc3.dat")},
{"numacc4", new StatTestData("./data/NIST/NumAcc4.dat")}
};
/// <summary>
/// Initializes a new instance of the StatisticsTests class.
/// </summary>
public StatisticsTests()
[Test]
public void ThrowsOnNullData()
{
var lottery = new StatTestData("./data/NIST/Lottery.dat");
_data.Add("lottery", lottery);
var lew = new StatTestData("./data/NIST/Lew.dat");
_data.Add("lew", lew);
var mavro = new StatTestData("./data/NIST/Mavro.dat");
_data.Add("mavro", mavro);
var michelso = new StatTestData("./data/NIST/Michelso.dat");
_data.Add("michelso", michelso);
var numacc1 = new StatTestData("./data/NIST/NumAcc1.dat");
_data.Add("numacc1", numacc1);
var numacc2 = new StatTestData("./data/NIST/NumAcc2.dat");
_data.Add("numacc2", numacc2);
var numacc3 = new StatTestData("./data/NIST/NumAcc3.dat");
_data.Add("numacc3", numacc3);
var numacc4 = new StatTestData("./data/NIST/NumAcc4.dat");
_data.Add("numacc4", numacc4);
double[] data = null;
Assert.Throws<ArgumentNullException>(() => Statistics.Minimum(data));
Assert.Throws<ArgumentNullException>(() => Statistics.Maximum(data));
Assert.Throws<ArgumentNullException>(() => Statistics.Mean(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.Minimum(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.Maximum(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.Median(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.LowerQuartile(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.UpperQuartile(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.Percentile(data, 30));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.Quantile(data, 0.3));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.QuantileCompatible(data, 0.3, QuantileCompatibility.Nearest));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.InterquartileRange(data));
Assert.Throws<ArgumentNullException>(() => SortedArrayStatistics.FiveNumberSummary(data));
Assert.Throws<ArgumentNullException>(() => ArrayStatistics.Minimum(data));
Assert.Throws<ArgumentNullException>(() => ArrayStatistics.Maximum(data));
Assert.Throws<ArgumentNullException>(() => ArrayStatistics.Mean(data));
Assert.Throws<ArgumentNullException>(() => StreamingStatistics.Minimum(data));
Assert.Throws<ArgumentNullException>(() => StreamingStatistics.Maximum(data));
}
[Test]
public void DoesNotThrowOnEmptyData()
{
double[] data = new double[0];
//Assert.DoesNotThrow(() => Statistics.Minimum(data));
//Assert.DoesNotThrow(() => Statistics.Maximum(data));
//Assert.DoesNotThrow(() => Statistics.Mean(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.Minimum(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.Maximum(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.Median(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.LowerQuartile(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.UpperQuartile(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.Percentile(data, 30));
Assert.DoesNotThrow(() => SortedArrayStatistics.Quantile(data, 0.3));
Assert.DoesNotThrow(() => SortedArrayStatistics.QuantileCompatible(data, 0.3, QuantileCompatibility.Nearest));
Assert.DoesNotThrow(() => SortedArrayStatistics.InterquartileRange(data));
Assert.DoesNotThrow(() => SortedArrayStatistics.FiveNumberSummary(data));
Assert.DoesNotThrow(() => ArrayStatistics.Minimum(data));
Assert.DoesNotThrow(() => ArrayStatistics.Maximum(data));
Assert.DoesNotThrow(() => ArrayStatistics.Mean(data));
Assert.DoesNotThrow(() => StreamingStatistics.Minimum(data));
Assert.DoesNotThrow(() => StreamingStatistics.Maximum(data));
}
/// <summary>
/// Validate mean.
/// </summary>
/// <param name="dataSet">Dataset name.</param>
[TestCase("lottery")]
[TestCase("lew")]
[TestCase("mavro")]
@ -86,17 +119,13 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
[TestCase("numacc2")]
[TestCase("numacc3")]
[TestCase("numacc4")]
public void Mean(string dataSet)
public void MeanConsistentWithNistData(string dataSet)
{
var data = _data[dataSet];
AssertHelpers.AlmostEqual(data.Mean, Statistics.Mean(data.Data), 15);
AssertHelpers.AlmostEqual(data.Mean, ArrayStatistics.Mean(data.Data), 15);
}
/// <summary>
/// <c>Nullable</c> mean.
/// </summary>
/// <param name="dataSet">Dataset name.</param>
[TestCase("lottery")]
[TestCase("lew")]
[TestCase("mavro")]
@ -105,22 +134,12 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
[TestCase("numacc2")]
[TestCase("numacc3")]
[TestCase("numacc4")]
public void NullableMean(string dataSet)
public void NullableMeanConsistentWithNistData(string dataSet)
{
var data = _data[dataSet];
AssertHelpers.AlmostEqual(data.Mean, Statistics.Mean(data.DataWithNulls), 15);
}
/// <summary>
/// Mean with <c>null</c> throws <c>ArgumentNullException</c>.
/// </summary>
[Test]
public void MeanThrowsArgumentNullException()
{
double[] data = null;
Assert.Throws<ArgumentNullException>(() => Statistics.Mean(data));
}
/// <summary>
/// Standard Deviation.
/// </summary>
@ -227,6 +246,20 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
AssertHelpers.AlmostEqual(2d, Statistics.StandardDeviation(gaussian.Samples().Take(10000)), 2);
}
[Test]
public void MinimumOfEmptyMustBeNaN()
{
Assert.That(StreamingStatistics.Minimum(new double[0]), Is.NaN);
Assert.That(StreamingStatistics.Minimum(new[] {2d }), Is.Not.NaN);
}
[Test]
public void MaximumOfEmptyMustBeNaN()
{
Assert.That(StreamingStatistics.Maximum(new double[0]), Is.NaN);
Assert.That(StreamingStatistics.Maximum(new[] { 2d }), Is.Not.NaN);
}
[Test]
public void SampleVarianceOfEmptyAndSingleMustBeNaN()
{
@ -257,5 +290,4 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
Assert.AreEqual(1.0, SortedArrayStatistics.Median(sorted));
}
}
#endif
}

Loading…
Cancel
Save