Browse Source

Statistics: single-pass MeanVariance method

optimization-1
Christoph Ruegg 13 years ago
parent
commit
5cb6db878f
  1. 12
      src/Numerics/Statistics/ArrayStatistics.cs
  2. 15
      src/Numerics/Statistics/Statistics.cs
  3. 79
      src/Numerics/Statistics/StreamingStatistics.cs
  4. 6
      src/UnitTests/StatisticsTests/StatisticsTests.cs

12
src/Numerics/Statistics/ArrayStatistics.cs

@ -172,6 +172,18 @@ namespace MathNet.Numerics.Statistics
return Math.Sqrt(PopulationVariance(population));
}
/// <summary>
/// Estimates the arithmetic sample mean and the unbiased population variance from the provided samples as unsorted array.
/// On a dataset of size N will use an N-1 normalizer (Bessel's correction).
/// Returns NaN for mean if data is empty or any entry is NaN and NaN for variance if data has less than two entries or if any entry is NaN.
/// </summary>
/// <param name="samples">Sample array, no sorting is assumed.</param>
public static Tuple<double, double> MeanVariance(double[] samples)
{
if (samples == null) throw new ArgumentNullException("samples");
return new Tuple<double, double>(Mean(samples), Variance(samples));
}
/// <summary>
/// Estimates the unbiased population covariance from the provided two sample arrays.
/// On a dataset of size N will use an N-1 normalizer (Bessel's correction).

15
src/Numerics/Statistics/Statistics.cs

@ -227,6 +227,21 @@ namespace MathNet.Numerics.Statistics
return StreamingStatistics.PopulationStandardDeviation(population.Where(d => d.HasValue).Select(d => d.Value));
}
/// <summary>
/// Estimates the sample mean and the unbiased population variance from the provided samples.
/// On a dataset of size N will use an N-1 normalizer (Bessel's correction).
/// Returns NaN for mean if data is empty or if any entry is NaN and NaN for variance if data has less than two entries or if any entry is NaN.
/// </summary>
/// <param name="samples">The data to calculate the mean of.</param>
/// <returns>The mean of the sample.</returns>
public static Tuple<double, double> MeanVariance(this IEnumerable<double> samples)
{
var array = samples as double[];
return array != null
? ArrayStatistics.MeanVariance(array)
: StreamingStatistics.MeanVariance(samples);
}
/// <summary>
/// Estimates the unbiased population covariance from the provided samples.
/// On a dataset of size N will use an N-1 normalizer (Bessel's correction).

79
src/Numerics/Statistics/StreamingStatistics.cs

@ -118,26 +118,28 @@ namespace MathNet.Numerics.Statistics
if (samples == null) throw new ArgumentNullException("samples");
double variance = 0;
double t = 0;
ulong j = 0;
double sum = 0;
ulong count = 0;
using (var iterator = samples.GetEnumerator())
{
if (iterator.MoveNext())
{
j++;
t = iterator.Current;
count++;
sum = iterator.Current;
}
while (iterator.MoveNext())
{
j++;
count++;
double xi = iterator.Current;
t += xi;
double diff = (j*xi) - t;
variance += (diff*diff)/(j*(j - 1));
sum += xi;
double diff = (count*xi) - sum;
variance += (diff*diff)/(count*(count - 1));
}
}
return j > 1 ? variance/(j - 1) : double.NaN;
return count > 1 ? variance/(count - 1) : double.NaN;
}
/// <summary>
@ -151,26 +153,28 @@ namespace MathNet.Numerics.Statistics
if (population == null) throw new ArgumentNullException("population");
double variance = 0;
double t = 0;
ulong j = 0;
double sum = 0;
ulong count = 0;
using (var iterator = population.GetEnumerator())
{
if (iterator.MoveNext())
{
j++;
t = iterator.Current;
count++;
sum = iterator.Current;
}
while (iterator.MoveNext())
{
j++;
count++;
double xi = iterator.Current;
t += xi;
double diff = (j*xi) - t;
variance += (diff*diff)/(j*(j - 1));
sum += xi;
double diff = (count*xi) - sum;
variance += (diff*diff)/(count*(count - 1));
}
}
return variance/j;
return variance/count;
}
/// <summary>
@ -195,6 +199,45 @@ namespace MathNet.Numerics.Statistics
return Math.Sqrt(PopulationVariance(population));
}
/// <summary>
/// Estimates the arithmetic sample mean and the unbiased population variance from the provided samples as enumerable sequence, in a single pass without memoization.
/// On a dataset of size N will use an N-1 normalizer (Bessel's correction).
/// Returns NaN for mean if data is empty or any entry is NaN, and NaN for variance if data has less than two entries or if any entry is NaN.
/// </summary>
/// <param name="samples">Sample stream, no sorting is assumed.</param>
public static Tuple<double, double> MeanVariance(IEnumerable<double> samples)
{
if (samples == null) throw new ArgumentNullException("samples");
double mean = 0;
double variance = 0;
double sum = 0;
ulong count = 0;
using (var iterator = samples.GetEnumerator())
{
if (iterator.MoveNext())
{
count++;
sum = mean = iterator.Current;
}
while (iterator.MoveNext())
{
count++;
double xi = iterator.Current;
sum += xi;
double diff = (count * xi) - sum;
variance += (diff * diff) / (count * (count - 1));
mean += (xi - mean) / count;
}
}
return new Tuple<double, double>(
count > 0 ? mean : double.NaN,
count > 1 ? variance/(count - 1) : double.NaN);
}
/// <summary>
/// Estimates the unbiased population covariance from the provided two sample enumerable sequences, in a single pass without memoization.
/// On a dataset of size N will use an N-1 normalizer (Bessel's correction).

6
src/UnitTests/StatisticsTests/StatisticsTests.cs

@ -179,6 +179,9 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
AssertHelpers.AlmostEqual(data.Mean, Statistics.Mean(data.Data), 15);
AssertHelpers.AlmostEqual(data.Mean, ArrayStatistics.Mean(data.Data), 15);
AssertHelpers.AlmostEqual(data.Mean, StreamingStatistics.Mean(data.Data), 15);
AssertHelpers.AlmostEqual(data.Mean, Statistics.MeanVariance(data.Data).Item1, 15);
AssertHelpers.AlmostEqual(data.Mean, ArrayStatistics.MeanVariance(data.Data).Item1, 15);
AssertHelpers.AlmostEqual(data.Mean, StreamingStatistics.MeanVariance(data.Data).Item1, 15);
}
[TestCase("lottery")]
@ -209,6 +212,9 @@ namespace MathNet.Numerics.UnitTests.StatisticsTests
AssertHelpers.AlmostEqual(data.StandardDeviation, Statistics.StandardDeviation(data.Data), digits);
AssertHelpers.AlmostEqual(data.StandardDeviation, ArrayStatistics.StandardDeviation(data.Data), digits);
AssertHelpers.AlmostEqual(data.StandardDeviation, StreamingStatistics.StandardDeviation(data.Data), digits);
AssertHelpers.AlmostEqual(data.StandardDeviation, Math.Sqrt(Statistics.MeanVariance(data.Data).Item2), digits);
AssertHelpers.AlmostEqual(data.StandardDeviation, Math.Sqrt(ArrayStatistics.MeanVariance(data.Data).Item2), digits);
AssertHelpers.AlmostEqual(data.StandardDeviation, Math.Sqrt(StreamingStatistics.MeanVariance(data.Data).Item2), digits);
}
[TestCase("lottery", 15)]

Loading…
Cancel
Save