From 79fa1236508f00e534dd1d2705414ee6150db62b Mon Sep 17 00:00:00 2001 From: Christoph Ruegg Date: Sat, 10 Feb 2018 17:58:46 +0100 Subject: [PATCH] Benchmarks: rework, cleanup --- build.fsx | 5 +- paket.dependencies | 1 - paket.lock | 4 - src/Benchmark/Benchmark.csproj | 6 +- .../LinearAlgebra/DenseMatrixProduct.cs | 91 +++++-------------- src/Benchmark/LinearAlgebra/DenseVectorAdd.cs | 7 ++ src/Benchmark/Program.cs | 43 ++++----- src/Benchmark/Providers.cs | 44 +++++++-- src/Benchmark/Transforms/FFT.cs | 69 ++++++++------ src/Benchmark/paket.references | 1 - src/Numerics.Tests/Numerics.Tests.CUDA.csproj | 2 +- src/Numerics.Tests/Numerics.Tests.MKL.csproj | 2 +- .../Numerics.Tests.OpenBLAS.csproj | 2 +- src/Numerics.Tests/Numerics.Tests.csproj | 2 +- 14 files changed, 130 insertions(+), 149 deletions(-) diff --git a/build.fsx b/build.fsx index 173e66b9..99ef5b42 100644 --- a/build.fsx +++ b/build.fsx @@ -316,9 +316,8 @@ let dataBundle = Target "Start" DoNothing Target "Clean" (fun _ -> - // Force delete the obj folder first (dotnet SDK has a habbit of fucking this folder up to a state where not even clean works...) - CleanDirs [ "src/Numerics/bin"; "src/FSharp/bin"; "src/TestData/bin"; "src/Numerics.Tests/bin"; "src/FSharp.Tests/bin"; "src/Data/Text/bin"; "src/Data/Matlab/bin"; "src/Data.Tests/bin" ] - CleanDirs [ "src/Numerics/obj"; "src/FSharp/obj"; "src/TestData/obj"; "src/Numerics.Tests/obj"; "src/FSharp.Tests/obj"; "src/Data/Text/obj"; "src/Data/Matlab/obj"; "src/Data.Tests/obj" ] + CleanDirs [ "src/Numerics/bin"; "src/FSharp/bin"; "src/TestData/bin"; "src/Numerics.Tests/bin"; "src/FSharp.Tests/bin"; "src/Data/Text/bin"; "src/Data/Matlab/bin"; "src/Data.Tests/bin"; "src/Benchmark/bin" ] + CleanDirs [ "src/Numerics/obj"; "src/FSharp/obj"; "src/TestData/obj"; "src/Numerics.Tests/obj"; "src/FSharp.Tests/obj"; "src/Data/Text/obj"; "src/Data/Matlab/obj"; "src/Data.Tests/obj"; "src/Benchmark/obj" ] CleanDirs [ "obj" ] CleanDirs [ "out/api"; "out/docs"; "out/packages" ] CleanDirs [ "out/lib" ] diff --git a/paket.dependencies b/paket.dependencies index 5c998ae8..374ab235 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -46,4 +46,3 @@ group Benchmark storage: none framework: net46 nuget BenchmarkDotNet content:none, strategy:min - nuget BenchmarkDotNet.Diagnostics.Windows content:none, strategy:min diff --git a/paket.lock b/paket.lock index 02be49ec..8493f320 100644 --- a/paket.lock +++ b/paket.lock @@ -1036,9 +1036,6 @@ NUGET System.Threading.Tasks.Extensions (>= 4.3) System.ValueTuple (>= 4.3) System.Xml.XmlSerializer (>= 4.3) - BenchmarkDotNet.Diagnostics.Windows (0.10.12) - content: none - BenchmarkDotNet (>= 0.10.12) - Microsoft.Diagnostics.Tracing.TraceEvent (>= 1.0.41) BenchmarkDotNet.Toolchains.Roslyn (0.10.12) - content: none BenchmarkDotNet.Core (>= 0.10.12) Microsoft.CodeAnalysis.CSharp (>= 2.4) @@ -1086,7 +1083,6 @@ NUGET System.Xml.XPath.XDocument (>= 4.3) Microsoft.CodeAnalysis.CSharp (2.4) - content: none Microsoft.CodeAnalysis.Common (2.4) - Microsoft.Diagnostics.Tracing.TraceEvent (1.0.41) - content: none Microsoft.DotNet.InternalAbstractions (1.0) - content: none Microsoft.DotNet.PlatformAbstractions (1.1.1) - content: none Microsoft.Win32.Registry (4.3) - content: none diff --git a/src/Benchmark/Benchmark.csproj b/src/Benchmark/Benchmark.csproj index a5c95e8f..6e8e8d83 100644 --- a/src/Benchmark/Benchmark.csproj +++ b/src/Benchmark/Benchmark.csproj @@ -2,8 +2,12 @@ Exe - net46 + 2.0.5 + false + Benchmark + Benchmark + false diff --git a/src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs b/src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs index 3cca4285..4d8caaaa 100644 --- a/src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs +++ b/src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs @@ -1,109 +1,62 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using BenchmarkDotNet.Attributes; -using MathNet.Numerics; using MathNet.Numerics.LinearAlgebra; -using MathNet.Numerics.Providers.LinearAlgebra; -using MathNet.Numerics.Providers.LinearAlgebra.Mkl; -using MathNet.Numerics.Threading; namespace Benchmark.LinearAlgebra { + + //BenchmarkRunner.Run(config); + //Benchmark(new DenseMatrixProduct(10,100), 100, "10 - 100x100 iterations"); + //Benchmark(new DenseMatrixProduct(25, 100), 100, "25 - 100x100 iterations"); + //Benchmark(new DenseMatrixProduct(50, 10), 100, "50 - 100x10 iterations"); + //Benchmark(new DenseMatrixProduct(100, 10), 100, "100 - 100x10 iterations"); + //Benchmark(new DenseMatrixProduct(250, 1), 10, "250 - 10x1 iterations"); + //Benchmark(new DenseMatrixProduct(500,1), 10, "500 - 10x1 iterations"); + //Benchmark(new DenseMatrixProduct(1000,1), 2, "1000 - 2x1 iterations"); + public class DenseMatrixProduct { readonly Dictionary> _data = new Dictionary>(); - readonly ILinearAlgebraProvider _mathnetMkl; - readonly ILinearAlgebraProvider _mathnetManaged; - //readonly ILinearAlgebraProvider _mathnetExperimental; - [Params(8, 64, 128)] public int M { get; set; } [Params(8, 64, 128)] public int N { get; set; } - static string Key(int m, int n) - { - return $"{m}x{n}"; - } + [Params(Provider.Managed, Provider.NativeMKLAutoHigh, Provider.NativeMKLAvx2High)] + public Provider Provider { get; set; } public DenseMatrixProduct() { - foreach (var m in new[] {8, 64, 128}) - foreach (var n in new[] {8, 64, 128}) + foreach (var m in new[] { 8, 64, 128 }) + foreach (var n in new[] { 8, 64, 128 }) { var key = Key(m, n); _data[key] = Matrix.Build.Random(m, n); } - - Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\"; - _mathnetMkl = LinearAlgebraControl.CreateNativeMKL(); - _mathnetManaged = LinearAlgebraControl.CreateManaged(); - //_mathnetExperimental = new ManagedLinearAlgebraProvider(Variation.Experimental); - - _mathnetMkl.InitializeVerify(); - _mathnetManaged.InitializeVerify(); - //_mathnetExperimental.InitializeVerify(); } - public void Verify() + static string Key(int m, int n) { - M = 8; - N = 8; - var resultMkl = MathNetMKL().ToRowArrays(); - var resultManaged = MathNetManaged().ToRowArrays(); - //var resultExperimental = MathNetExperimental().ToRowArrays(); - for (int i = 0; i < 8; i++) - { - for (int j = 0; j < 8; j++) - { - if (!resultMkl[i][j].AlmostEqual(resultManaged[i][j], 1e-14)) - { - throw new Exception($"Managed [{i}][{j}] {resultManaged[i][j]} != {resultMkl[i][j]}"); - } - //if (!resultMkl[i][j].AlmostEqual(resultExperimental[i][j], 1e-14)) - //{ - // throw new Exception($"Experimental [{i}][{j}] {resultExperimental[i][j]} != {resultMkl[i][j]}"); - //} - } - } + return $"{m}x{n}"; } - [Benchmark(OperationsPerInvoke = 1)] - public Matrix MathNetMKL() + [GlobalSetup] + public void Setup() { - LinearAlgebraControl.Provider = _mathnetMkl; - if (M != N) - { - return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]); - } - - return _data[Key(M, N)]*_data[Key(M, N)]; + Providers.ForceProvider(Provider); } [Benchmark(OperationsPerInvoke = 1)] - public Matrix MathNetManaged() + public Matrix Product() { - LinearAlgebraControl.Provider = _mathnetManaged; if (M != N) { return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]); } - return _data[Key(M, N)] * _data[Key(M, N)]; + return _data[Key(M, N)]*_data[Key(M, N)]; } - - //[Benchmark(OperationsPerInvoke = 1)] - //public Matrix MathNetExperimental() - //{ - // LinearAlgebraControl.Provider = _mathnetExperimental; - // if (M != N) - // { - // return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]); - // } - - // return _data[Key(M, N)] * _data[Key(M, N)]; - //} } } diff --git a/src/Benchmark/LinearAlgebra/DenseVectorAdd.cs b/src/Benchmark/LinearAlgebra/DenseVectorAdd.cs index c74efebf..ebf42170 100644 --- a/src/Benchmark/LinearAlgebra/DenseVectorAdd.cs +++ b/src/Benchmark/LinearAlgebra/DenseVectorAdd.cs @@ -8,6 +8,13 @@ using MathNet.Numerics.Threading; namespace Benchmark.LinearAlgebra { + + +//Benchmark(new LinearAlgebra.DenseVectorAdd(10000000,1), 10, "Large (10'000'000) - 10x1 iterations"); +//Benchmark(new LinearAlgebra.DenseVectorAdd(100,1000), 100, "Small (100) - 100x1000 iterations"); + + + // public class DenseVectorAdd // { // readonly int _rounds; diff --git a/src/Benchmark/Program.cs b/src/Benchmark/Program.cs index 63887169..6d97bdc8 100644 --- a/src/Benchmark/Program.cs +++ b/src/Benchmark/Program.cs @@ -1,42 +1,31 @@ using System; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnostics.Windows; -using BenchmarkDotNet.Jobs; +using System.Linq; using BenchmarkDotNet.Running; using MathNet.Numerics; +using MathNet.Numerics.Providers.FourierTransform; +using MathNet.Numerics.Providers.LinearAlgebra; namespace Benchmark { public class Program { - public static void Main() + public static void Main(string[] args) { - Providers.ForceNativeMKL(); Console.WriteLine(Control.Describe()); + foreach (var provider in Enum.GetValues(typeof(Provider)).Cast()) + { + Providers.ForceProvider(provider); + Console.WriteLine($"{provider}: LA={LinearAlgebraControl.Provider}, FFT={FourierTransformControl.Provider}"); + } - var subject = new LinearAlgebra.DenseMatrixProduct(); - subject.Verify(); - Console.WriteLine("Verified."); + var switcher = new BenchmarkSwitcher( + new[] + { + typeof(Transforms.FFT), + typeof(LinearAlgebra.DenseMatrixProduct), + }); - var config = ManualConfig.Create(DefaultConfig.Instance); - config.Add(Job.RyuJitX64, Job.LegacyJitX86); - //config.Add(new MemoryDiagnoser()); - - BenchmarkRunner.Run(config); - //BenchmarkRunner.Run(config); - - //Benchmark(new LinearAlgebra.DenseVectorAdd(10000000,1), 10, "Large (10'000'000) - 10x1 iterations"); - //Benchmark(new LinearAlgebra.DenseVectorAdd(100,1000), 100, "Small (100) - 100x1000 iterations"); - - //DenseMatrixProduct.Verify(5); - //DenseMatrixProduct.Verify(100); - //Benchmark(new DenseMatrixProduct(10,100), 100, "10 - 100x100 iterations"); - //Benchmark(new DenseMatrixProduct(25, 100), 100, "25 - 100x100 iterations"); - //Benchmark(new DenseMatrixProduct(50, 10), 100, "50 - 100x10 iterations"); - //Benchmark(new DenseMatrixProduct(100, 10), 100, "100 - 100x10 iterations"); - //Benchmark(new DenseMatrixProduct(250, 1), 10, "250 - 10x1 iterations"); - //Benchmark(new DenseMatrixProduct(500,1), 10, "500 - 10x1 iterations"); - //Benchmark(new DenseMatrixProduct(1000,1), 2, "1000 - 2x1 iterations"); + switcher.Run(args); } } } diff --git a/src/Benchmark/Providers.cs b/src/Benchmark/Providers.cs index e20368d4..d9461457 100644 --- a/src/Benchmark/Providers.cs +++ b/src/Benchmark/Providers.cs @@ -1,21 +1,45 @@ using MathNet.Numerics; +using MathNet.Numerics.Providers.Common.Mkl; namespace Benchmark { + public enum Provider : int + { + Managed = 0, + NativeMKLAutoHigh = 1, + NativeMKLAutoLow = 2, + NativeMKLAvx2High = 3, + NativeMKLAvx2Low = 4, + NativeOpenBLAS = 5 + } + public static class Providers { - public static void ForceNativeMKL() + public static void ForceProvider(Provider provider) { - //Control.NativeProviderPath = @"C:\Triage\NATIVE-Win\"; - Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\"; - Control.UseNativeMKL(); - } + //Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\"; - public static void ForceOpenBLAS() - { - //Control.NativeProviderPath = @"C:\Triage\NATIVE-Win\"; - Control.NativeProviderPath = @"..\..\..\..\out\OpenBLAS\Windows\"; - Control.UseNativeOpenBLAS(); + switch (provider) + { + case Provider.Managed: + Control.UseManaged(); + break; + case Provider.NativeMKLAutoHigh: + Control.UseNativeMKL(MklConsistency.Auto, MklPrecision.Double, MklAccuracy.High); + break; + case Provider.NativeMKLAutoLow: + Control.UseNativeMKL(MklConsistency.Auto, MklPrecision.Double, MklAccuracy.Low); + break; + case Provider.NativeMKLAvx2High: + Control.UseNativeMKL(MklConsistency.AVX2, MklPrecision.Double, MklAccuracy.High); + break; + case Provider.NativeMKLAvx2Low: + Control.UseNativeMKL(MklConsistency.AVX2, MklPrecision.Double, MklAccuracy.Low); + break; + case Provider.NativeOpenBLAS: + Control.UseNativeOpenBLAS(); + break; + } } } } diff --git a/src/Benchmark/Transforms/FFT.cs b/src/Benchmark/Transforms/FFT.cs index 39af2d6c..133a8a8f 100644 --- a/src/Benchmark/Transforms/FFT.cs +++ b/src/Benchmark/Transforms/FFT.cs @@ -1,51 +1,62 @@ -using System; -using System.Collections.Generic; -using System.Numerics; +using System.Numerics; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; using MathNet.Numerics; using MathNet.Numerics.IntegralTransforms; namespace Benchmark.Transforms { + [Config(typeof(Config))] public class FFT { - readonly Dictionary _data = new Dictionary(); - - [Params(32, 64, 128, 1024, 8192, 65536)] - public int N { get; set; } - - public FFT() + class Config : ManualConfig { - var realSinusoidal = Generate.Sinusoidal(65536, 32, -2.0, 2.0); - var imagSawtooth = Generate.Sawtooth(65536, 32, -20.0, 20.0); - var signal = Generate.Map2(realSinusoidal, imagSawtooth, (r, i) => new Complex(r, i)); - foreach (var n in new[] { 32, 64, 128, 1024, 8192, 65536 }) + public Config() { - var s = new Complex[n]; - Array.Copy(signal, 0, s, 0, n); - _data[n] = s; + Add( + new Job("CLR RyuJit x64", RunMode.Default, EnvMode.RyuJitX64) + { + Env = { Runtime = Runtime.Clr, Platform = Platform.X64 } + }, + new Job("CLR RyuJit x86", RunMode.Default, EnvMode.RyuJitX86) + { + Env = { Runtime = Runtime.Clr, Platform = Platform.X86 } + }, + new Job("CLR LegacyJit x64", RunMode.Default, EnvMode.LegacyJitX64) + { + Env = { Runtime = Runtime.Clr, Platform = Platform.X64 } + }, + new Job("CLR LegacyJit x86", RunMode.Default, EnvMode.LegacyJitX86) + { + Env = { Runtime = Runtime.Clr, Platform = Platform.X86 } + }); } - - Providers.ForceNativeMKL(); } - [Setup] - public void Setup() - { - } + [Params(32, 128)] //, 64, 1024, 8192, 65536)] + public int N { get; set; } + + [Params(Provider.Managed, Provider.NativeMKLAutoHigh, Provider.NativeMKLAvx2High)] + public Provider Provider { get; set; } - [Benchmark(Baseline = true, OperationsPerInvoke = 2)] - public void Managed() + Complex[] _data; + + [GlobalSetup] + public void Setup() { - Fourier.Radix2Forward(_data[N], FourierOptions.NoScaling); - Fourier.Radix2Inverse(_data[N], FourierOptions.NoScaling); + Providers.ForceProvider(Provider); + var realSinusoidal = Generate.Sinusoidal(N, 32, -2.0, 2.0); + var imagSawtooth = Generate.Sawtooth(N, 32, -20.0, 20.0); + _data = Generate.Map2(realSinusoidal, imagSawtooth, (r, i) => new Complex(r, i)); } [Benchmark(OperationsPerInvoke = 2)] - public void NativeMKL() + public void Transform() { - Fourier.Forward(_data[N], FourierOptions.NoScaling); - Fourier.Inverse(_data[N], FourierOptions.NoScaling); + Fourier.Forward(_data, FourierOptions.NoScaling); + Fourier.Inverse(_data, FourierOptions.NoScaling); } } } diff --git a/src/Benchmark/paket.references b/src/Benchmark/paket.references index cf8bb8a5..41438b03 100644 --- a/src/Benchmark/paket.references +++ b/src/Benchmark/paket.references @@ -1,3 +1,2 @@ group Benchmark BenchmarkDotNet - BenchmarkDotNet.Diagnostics.Windows diff --git a/src/Numerics.Tests/Numerics.Tests.CUDA.csproj b/src/Numerics.Tests/Numerics.Tests.CUDA.csproj index 79fd60c6..61158f32 100644 --- a/src/Numerics.Tests/Numerics.Tests.CUDA.csproj +++ b/src/Numerics.Tests/Numerics.Tests.CUDA.csproj @@ -3,7 +3,7 @@ Exe net45;netcoreapp1.1;netcoreapp2.0 - 2.0.0 + 2.0.5 false MathNet.Numerics.Tests.CUDA MathNet.Numerics.Tests diff --git a/src/Numerics.Tests/Numerics.Tests.MKL.csproj b/src/Numerics.Tests/Numerics.Tests.MKL.csproj index 27d8264e..2be9faa1 100644 --- a/src/Numerics.Tests/Numerics.Tests.MKL.csproj +++ b/src/Numerics.Tests/Numerics.Tests.MKL.csproj @@ -3,7 +3,7 @@ Exe net45;netcoreapp1.1;netcoreapp2.0 - 2.0.0 + 2.0.5 false MathNet.Numerics.Tests.MKL MathNet.Numerics.Tests diff --git a/src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj b/src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj index 88894a04..3abe01e7 100644 --- a/src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj +++ b/src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj @@ -3,7 +3,7 @@ Exe net45;netcoreapp1.1;netcoreapp2.0 - 2.0.0 + 2.0.5 false MathNet.Numerics.Tests.OpenBLAS MathNet.Numerics.Tests diff --git a/src/Numerics.Tests/Numerics.Tests.csproj b/src/Numerics.Tests/Numerics.Tests.csproj index 97cc59c2..6cdd42e3 100644 --- a/src/Numerics.Tests/Numerics.Tests.csproj +++ b/src/Numerics.Tests/Numerics.Tests.csproj @@ -3,7 +3,7 @@ Exe net45;netcoreapp1.1;netcoreapp2.0 - 2.0.0 + 2.0.5 false MathNet.Numerics.Tests MathNet.Numerics.Tests