Browse Source

Benchmarks: rework, cleanup

spatial
Christoph Ruegg 9 years ago
parent
commit
79fa123650
  1. 5
      build.fsx
  2. 1
      paket.dependencies
  3. 4
      paket.lock
  4. 6
      src/Benchmark/Benchmark.csproj
  5. 91
      src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs
  6. 7
      src/Benchmark/LinearAlgebra/DenseVectorAdd.cs
  7. 43
      src/Benchmark/Program.cs
  8. 44
      src/Benchmark/Providers.cs
  9. 69
      src/Benchmark/Transforms/FFT.cs
  10. 1
      src/Benchmark/paket.references
  11. 2
      src/Numerics.Tests/Numerics.Tests.CUDA.csproj
  12. 2
      src/Numerics.Tests/Numerics.Tests.MKL.csproj
  13. 2
      src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj
  14. 2
      src/Numerics.Tests/Numerics.Tests.csproj

5
build.fsx

@ -316,9 +316,8 @@ let dataBundle =
Target "Start" DoNothing
Target "Clean" (fun _ ->
// Force delete the obj folder first (dotnet SDK has a habbit of fucking this folder up to a state where not even clean works...)
CleanDirs [ "src/Numerics/bin"; "src/FSharp/bin"; "src/TestData/bin"; "src/Numerics.Tests/bin"; "src/FSharp.Tests/bin"; "src/Data/Text/bin"; "src/Data/Matlab/bin"; "src/Data.Tests/bin" ]
CleanDirs [ "src/Numerics/obj"; "src/FSharp/obj"; "src/TestData/obj"; "src/Numerics.Tests/obj"; "src/FSharp.Tests/obj"; "src/Data/Text/obj"; "src/Data/Matlab/obj"; "src/Data.Tests/obj" ]
CleanDirs [ "src/Numerics/bin"; "src/FSharp/bin"; "src/TestData/bin"; "src/Numerics.Tests/bin"; "src/FSharp.Tests/bin"; "src/Data/Text/bin"; "src/Data/Matlab/bin"; "src/Data.Tests/bin"; "src/Benchmark/bin" ]
CleanDirs [ "src/Numerics/obj"; "src/FSharp/obj"; "src/TestData/obj"; "src/Numerics.Tests/obj"; "src/FSharp.Tests/obj"; "src/Data/Text/obj"; "src/Data/Matlab/obj"; "src/Data.Tests/obj"; "src/Benchmark/obj" ]
CleanDirs [ "obj" ]
CleanDirs [ "out/api"; "out/docs"; "out/packages" ]
CleanDirs [ "out/lib" ]

1
paket.dependencies

@ -46,4 +46,3 @@ group Benchmark
storage: none
framework: net46
nuget BenchmarkDotNet content:none, strategy:min
nuget BenchmarkDotNet.Diagnostics.Windows content:none, strategy:min

4
paket.lock

@ -1036,9 +1036,6 @@ NUGET
System.Threading.Tasks.Extensions (>= 4.3)
System.ValueTuple (>= 4.3)
System.Xml.XmlSerializer (>= 4.3)
BenchmarkDotNet.Diagnostics.Windows (0.10.12) - content: none
BenchmarkDotNet (>= 0.10.12)
Microsoft.Diagnostics.Tracing.TraceEvent (>= 1.0.41)
BenchmarkDotNet.Toolchains.Roslyn (0.10.12) - content: none
BenchmarkDotNet.Core (>= 0.10.12)
Microsoft.CodeAnalysis.CSharp (>= 2.4)
@ -1086,7 +1083,6 @@ NUGET
System.Xml.XPath.XDocument (>= 4.3)
Microsoft.CodeAnalysis.CSharp (2.4) - content: none
Microsoft.CodeAnalysis.Common (2.4)
Microsoft.Diagnostics.Tracing.TraceEvent (1.0.41) - content: none
Microsoft.DotNet.InternalAbstractions (1.0) - content: none
Microsoft.DotNet.PlatformAbstractions (1.1.1) - content: none
Microsoft.Win32.Registry (4.3) - content: none

6
src/Benchmark/Benchmark.csproj

@ -2,8 +2,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<!--<TargetFrameworks>net46;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>-->
<TargetFrameworks>net46</TargetFrameworks>
<RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AssemblyName>Benchmark</AssemblyName>
<RootNamespace>Benchmark</RootNamespace>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Numerics\Numerics.csproj" />

91
src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs

@ -1,109 +1,62 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using BenchmarkDotNet.Attributes;
using MathNet.Numerics;
using MathNet.Numerics.LinearAlgebra;
using MathNet.Numerics.Providers.LinearAlgebra;
using MathNet.Numerics.Providers.LinearAlgebra.Mkl;
using MathNet.Numerics.Threading;
namespace Benchmark.LinearAlgebra
{
//BenchmarkRunner.Run<LinearAlgebra.DenseMatrixProduct>(config);
//Benchmark(new DenseMatrixProduct(10,100), 100, "10 - 100x100 iterations");
//Benchmark(new DenseMatrixProduct(25, 100), 100, "25 - 100x100 iterations");
//Benchmark(new DenseMatrixProduct(50, 10), 100, "50 - 100x10 iterations");
//Benchmark(new DenseMatrixProduct(100, 10), 100, "100 - 100x10 iterations");
//Benchmark(new DenseMatrixProduct(250, 1), 10, "250 - 10x1 iterations");
//Benchmark(new DenseMatrixProduct(500,1), 10, "500 - 10x1 iterations");
//Benchmark(new DenseMatrixProduct(1000,1), 2, "1000 - 2x1 iterations");
public class DenseMatrixProduct
{
readonly Dictionary<string, Matrix<double>> _data = new Dictionary<string, Matrix<double>>();
readonly ILinearAlgebraProvider _mathnetMkl;
readonly ILinearAlgebraProvider _mathnetManaged;
//readonly ILinearAlgebraProvider _mathnetExperimental;
[Params(8, 64, 128)]
public int M { get; set; }
[Params(8, 64, 128)]
public int N { get; set; }
static string Key(int m, int n)
{
return $"{m}x{n}";
}
[Params(Provider.Managed, Provider.NativeMKLAutoHigh, Provider.NativeMKLAvx2High)]
public Provider Provider { get; set; }
public DenseMatrixProduct()
{
foreach (var m in new[] {8, 64, 128})
foreach (var n in new[] {8, 64, 128})
foreach (var m in new[] { 8, 64, 128 })
foreach (var n in new[] { 8, 64, 128 })
{
var key = Key(m, n);
_data[key] = Matrix<double>.Build.Random(m, n);
}
Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
_mathnetMkl = LinearAlgebraControl.CreateNativeMKL();
_mathnetManaged = LinearAlgebraControl.CreateManaged();
//_mathnetExperimental = new ManagedLinearAlgebraProvider(Variation.Experimental);
_mathnetMkl.InitializeVerify();
_mathnetManaged.InitializeVerify();
//_mathnetExperimental.InitializeVerify();
}
public void Verify()
static string Key(int m, int n)
{
M = 8;
N = 8;
var resultMkl = MathNetMKL().ToRowArrays();
var resultManaged = MathNetManaged().ToRowArrays();
//var resultExperimental = MathNetExperimental().ToRowArrays();
for (int i = 0; i < 8; i++)
{
for (int j = 0; j < 8; j++)
{
if (!resultMkl[i][j].AlmostEqual(resultManaged[i][j], 1e-14))
{
throw new Exception($"Managed [{i}][{j}] {resultManaged[i][j]} != {resultMkl[i][j]}");
}
//if (!resultMkl[i][j].AlmostEqual(resultExperimental[i][j], 1e-14))
//{
// throw new Exception($"Experimental [{i}][{j}] {resultExperimental[i][j]} != {resultMkl[i][j]}");
//}
}
}
return $"{m}x{n}";
}
[Benchmark(OperationsPerInvoke = 1)]
public Matrix<double> MathNetMKL()
[GlobalSetup]
public void Setup()
{
LinearAlgebraControl.Provider = _mathnetMkl;
if (M != N)
{
return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
}
return _data[Key(M, N)]*_data[Key(M, N)];
Providers.ForceProvider(Provider);
}
[Benchmark(OperationsPerInvoke = 1)]
public Matrix<double> MathNetManaged()
public Matrix<double> Product()
{
LinearAlgebraControl.Provider = _mathnetManaged;
if (M != N)
{
return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
}
return _data[Key(M, N)] * _data[Key(M, N)];
return _data[Key(M, N)]*_data[Key(M, N)];
}
//[Benchmark(OperationsPerInvoke = 1)]
//public Matrix<double> MathNetExperimental()
//{
// LinearAlgebraControl.Provider = _mathnetExperimental;
// if (M != N)
// {
// return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
// }
// return _data[Key(M, N)] * _data[Key(M, N)];
//}
}
}

7
src/Benchmark/LinearAlgebra/DenseVectorAdd.cs

@ -8,6 +8,13 @@ using MathNet.Numerics.Threading;
namespace Benchmark.LinearAlgebra
{
//Benchmark(new LinearAlgebra.DenseVectorAdd(10000000,1), 10, "Large (10'000'000) - 10x1 iterations");
//Benchmark(new LinearAlgebra.DenseVectorAdd(100,1000), 100, "Small (100) - 100x1000 iterations");
// public class DenseVectorAdd
// {
// readonly int _rounds;

43
src/Benchmark/Program.cs

@ -1,42 +1,31 @@
using System;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnostics.Windows;
using BenchmarkDotNet.Jobs;
using System.Linq;
using BenchmarkDotNet.Running;
using MathNet.Numerics;
using MathNet.Numerics.Providers.FourierTransform;
using MathNet.Numerics.Providers.LinearAlgebra;
namespace Benchmark
{
public class Program
{
public static void Main()
public static void Main(string[] args)
{
Providers.ForceNativeMKL();
Console.WriteLine(Control.Describe());
foreach (var provider in Enum.GetValues(typeof(Provider)).Cast<Provider>())
{
Providers.ForceProvider(provider);
Console.WriteLine($"{provider}: LA={LinearAlgebraControl.Provider}, FFT={FourierTransformControl.Provider}");
}
var subject = new LinearAlgebra.DenseMatrixProduct();
subject.Verify();
Console.WriteLine("Verified.");
var switcher = new BenchmarkSwitcher(
new[]
{
typeof(Transforms.FFT),
typeof(LinearAlgebra.DenseMatrixProduct),
});
var config = ManualConfig.Create(DefaultConfig.Instance);
config.Add(Job.RyuJitX64, Job.LegacyJitX86);
//config.Add(new MemoryDiagnoser());
BenchmarkRunner.Run<Transforms.FFT>(config);
//BenchmarkRunner.Run<LinearAlgebra.DenseMatrixProduct>(config);
//Benchmark(new LinearAlgebra.DenseVectorAdd(10000000,1), 10, "Large (10'000'000) - 10x1 iterations");
//Benchmark(new LinearAlgebra.DenseVectorAdd(100,1000), 100, "Small (100) - 100x1000 iterations");
//DenseMatrixProduct.Verify(5);
//DenseMatrixProduct.Verify(100);
//Benchmark(new DenseMatrixProduct(10,100), 100, "10 - 100x100 iterations");
//Benchmark(new DenseMatrixProduct(25, 100), 100, "25 - 100x100 iterations");
//Benchmark(new DenseMatrixProduct(50, 10), 100, "50 - 100x10 iterations");
//Benchmark(new DenseMatrixProduct(100, 10), 100, "100 - 100x10 iterations");
//Benchmark(new DenseMatrixProduct(250, 1), 10, "250 - 10x1 iterations");
//Benchmark(new DenseMatrixProduct(500,1), 10, "500 - 10x1 iterations");
//Benchmark(new DenseMatrixProduct(1000,1), 2, "1000 - 2x1 iterations");
switcher.Run(args);
}
}
}

44
src/Benchmark/Providers.cs

@ -1,21 +1,45 @@
using MathNet.Numerics;
using MathNet.Numerics.Providers.Common.Mkl;
namespace Benchmark
{
public enum Provider : int
{
Managed = 0,
NativeMKLAutoHigh = 1,
NativeMKLAutoLow = 2,
NativeMKLAvx2High = 3,
NativeMKLAvx2Low = 4,
NativeOpenBLAS = 5
}
public static class Providers
{
public static void ForceNativeMKL()
public static void ForceProvider(Provider provider)
{
//Control.NativeProviderPath = @"C:\Triage\NATIVE-Win\";
Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
Control.UseNativeMKL();
}
//Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
public static void ForceOpenBLAS()
{
//Control.NativeProviderPath = @"C:\Triage\NATIVE-Win\";
Control.NativeProviderPath = @"..\..\..\..\out\OpenBLAS\Windows\";
Control.UseNativeOpenBLAS();
switch (provider)
{
case Provider.Managed:
Control.UseManaged();
break;
case Provider.NativeMKLAutoHigh:
Control.UseNativeMKL(MklConsistency.Auto, MklPrecision.Double, MklAccuracy.High);
break;
case Provider.NativeMKLAutoLow:
Control.UseNativeMKL(MklConsistency.Auto, MklPrecision.Double, MklAccuracy.Low);
break;
case Provider.NativeMKLAvx2High:
Control.UseNativeMKL(MklConsistency.AVX2, MklPrecision.Double, MklAccuracy.High);
break;
case Provider.NativeMKLAvx2Low:
Control.UseNativeMKL(MklConsistency.AVX2, MklPrecision.Double, MklAccuracy.Low);
break;
case Provider.NativeOpenBLAS:
Control.UseNativeOpenBLAS();
break;
}
}
}
}

69
src/Benchmark/Transforms/FFT.cs

@ -1,51 +1,62 @@
using System;
using System.Collections.Generic;
using System.Numerics;
using System.Numerics;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using MathNet.Numerics;
using MathNet.Numerics.IntegralTransforms;
namespace Benchmark.Transforms
{
[Config(typeof(Config))]
public class FFT
{
readonly Dictionary<int, Complex[]> _data = new Dictionary<int, Complex[]>();
[Params(32, 64, 128, 1024, 8192, 65536)]
public int N { get; set; }
public FFT()
class Config : ManualConfig
{
var realSinusoidal = Generate.Sinusoidal(65536, 32, -2.0, 2.0);
var imagSawtooth = Generate.Sawtooth(65536, 32, -20.0, 20.0);
var signal = Generate.Map2(realSinusoidal, imagSawtooth, (r, i) => new Complex(r, i));
foreach (var n in new[] { 32, 64, 128, 1024, 8192, 65536 })
public Config()
{
var s = new Complex[n];
Array.Copy(signal, 0, s, 0, n);
_data[n] = s;
Add(
new Job("CLR RyuJit x64", RunMode.Default, EnvMode.RyuJitX64)
{
Env = { Runtime = Runtime.Clr, Platform = Platform.X64 }
},
new Job("CLR RyuJit x86", RunMode.Default, EnvMode.RyuJitX86)
{
Env = { Runtime = Runtime.Clr, Platform = Platform.X86 }
},
new Job("CLR LegacyJit x64", RunMode.Default, EnvMode.LegacyJitX64)
{
Env = { Runtime = Runtime.Clr, Platform = Platform.X64 }
},
new Job("CLR LegacyJit x86", RunMode.Default, EnvMode.LegacyJitX86)
{
Env = { Runtime = Runtime.Clr, Platform = Platform.X86 }
});
}
Providers.ForceNativeMKL();
}
[Setup]
public void Setup()
{
}
[Params(32, 128)] //, 64, 1024, 8192, 65536)]
public int N { get; set; }
[Params(Provider.Managed, Provider.NativeMKLAutoHigh, Provider.NativeMKLAvx2High)]
public Provider Provider { get; set; }
[Benchmark(Baseline = true, OperationsPerInvoke = 2)]
public void Managed()
Complex[] _data;
[GlobalSetup]
public void Setup()
{
Fourier.Radix2Forward(_data[N], FourierOptions.NoScaling);
Fourier.Radix2Inverse(_data[N], FourierOptions.NoScaling);
Providers.ForceProvider(Provider);
var realSinusoidal = Generate.Sinusoidal(N, 32, -2.0, 2.0);
var imagSawtooth = Generate.Sawtooth(N, 32, -20.0, 20.0);
_data = Generate.Map2(realSinusoidal, imagSawtooth, (r, i) => new Complex(r, i));
}
[Benchmark(OperationsPerInvoke = 2)]
public void NativeMKL()
public void Transform()
{
Fourier.Forward(_data[N], FourierOptions.NoScaling);
Fourier.Inverse(_data[N], FourierOptions.NoScaling);
Fourier.Forward(_data, FourierOptions.NoScaling);
Fourier.Inverse(_data, FourierOptions.NoScaling);
}
}
}

1
src/Benchmark/paket.references

@ -1,3 +1,2 @@
group Benchmark
BenchmarkDotNet
BenchmarkDotNet.Diagnostics.Windows

2
src/Numerics.Tests/Numerics.Tests.CUDA.csproj

@ -3,7 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
<RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
<RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AssemblyName>MathNet.Numerics.Tests.CUDA</AssemblyName>
<RootNamespace>MathNet.Numerics.Tests</RootNamespace>

2
src/Numerics.Tests/Numerics.Tests.MKL.csproj

@ -3,7 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
<RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
<RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AssemblyName>MathNet.Numerics.Tests.MKL</AssemblyName>
<RootNamespace>MathNet.Numerics.Tests</RootNamespace>

2
src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj

@ -3,7 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
<RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
<RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AssemblyName>MathNet.Numerics.Tests.OpenBLAS</AssemblyName>
<RootNamespace>MathNet.Numerics.Tests</RootNamespace>

2
src/Numerics.Tests/Numerics.Tests.csproj

@ -3,7 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
<RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
<RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AssemblyName>MathNet.Numerics.Tests</AssemblyName>
<RootNamespace>MathNet.Numerics.Tests</RootNamespace>

Loading…
Cancel
Save