Benchmarks: rework, cleanup

build.fsx

@@ -316,9 +316,8 @@ let dataBundle =
 Target "Start" DoNothing
 
 Target "Clean" (fun _ ->
-    // Force delete the obj folder first (dotnet SDK has a habbit of fucking this folder up to a state where not even clean works...)
+    CleanDirs [ "src/Numerics/bin"; "src/FSharp/bin"; "src/TestData/bin"; "src/Numerics.Tests/bin"; "src/FSharp.Tests/bin"; "src/Data/Text/bin"; "src/Data/Matlab/bin"; "src/Data.Tests/bin"; "src/Benchmark/bin" ]
-    CleanDirs [ "src/Numerics/bin"; "src/FSharp/bin"; "src/TestData/bin"; "src/Numerics.Tests/bin"; "src/FSharp.Tests/bin"; "src/Data/Text/bin"; "src/Data/Matlab/bin"; "src/Data.Tests/bin" ]
+    CleanDirs [ "src/Numerics/obj"; "src/FSharp/obj"; "src/TestData/obj"; "src/Numerics.Tests/obj"; "src/FSharp.Tests/obj"; "src/Data/Text/obj"; "src/Data/Matlab/obj"; "src/Data.Tests/obj"; "src/Benchmark/obj" ]
-    CleanDirs [ "src/Numerics/obj"; "src/FSharp/obj"; "src/TestData/obj"; "src/Numerics.Tests/obj"; "src/FSharp.Tests/obj"; "src/Data/Text/obj"; "src/Data/Matlab/obj"; "src/Data.Tests/obj" ]
     CleanDirs [ "obj" ]
     CleanDirs [ "out/api"; "out/docs"; "out/packages" ]
     CleanDirs [ "out/lib" ]

paket.dependencies

@@ -46,4 +46,3 @@ group Benchmark
     storage: none
     framework: net46
     nuget BenchmarkDotNet content:none, strategy:min
-    nuget BenchmarkDotNet.Diagnostics.Windows content:none, strategy:min

paket.lock

@@ -1036,9 +1036,6 @@ NUGET
       System.Threading.Tasks.Extensions (>= 4.3)
       System.ValueTuple (>= 4.3)
       System.Xml.XmlSerializer (>= 4.3)
-    BenchmarkDotNet.Diagnostics.Windows (0.10.12) - content: none
-      BenchmarkDotNet (>= 0.10.12)
-      Microsoft.Diagnostics.Tracing.TraceEvent (>= 1.0.41)
     BenchmarkDotNet.Toolchains.Roslyn (0.10.12) - content: none
       BenchmarkDotNet.Core (>= 0.10.12)
       Microsoft.CodeAnalysis.CSharp (>= 2.4)
@@ -1086,7 +1083,6 @@ NUGET
       System.Xml.XPath.XDocument (>= 4.3)
     Microsoft.CodeAnalysis.CSharp (2.4) - content: none
       Microsoft.CodeAnalysis.Common (2.4)
-    Microsoft.Diagnostics.Tracing.TraceEvent (1.0.41) - content: none
     Microsoft.DotNet.InternalAbstractions (1.0) - content: none
     Microsoft.DotNet.PlatformAbstractions (1.1.1) - content: none
     Microsoft.Win32.Registry (4.3) - content: none

src/Benchmark/Benchmark.csproj

@@ -2,8 +2,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <!--<TargetFrameworks>net46;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>-->
     <TargetFrameworks>net46</TargetFrameworks>
+    <RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <AssemblyName>Benchmark</AssemblyName>
+    <RootNamespace>Benchmark</RootNamespace>
+    <IsPackable>false</IsPackable>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\Numerics\Numerics.csproj" />

src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs

@@ -1,109 +1,62 @@
-using System;
+using System.Collections.Generic;
-using System.Collections.Generic;
 using BenchmarkDotNet.Attributes;
-using MathNet.Numerics;
 using MathNet.Numerics.LinearAlgebra;
-using MathNet.Numerics.Providers.LinearAlgebra;
-using MathNet.Numerics.Providers.LinearAlgebra.Mkl;
-using MathNet.Numerics.Threading;
 
 namespace Benchmark.LinearAlgebra
 {
+
+    //BenchmarkRunner.Run<LinearAlgebra.DenseMatrixProduct>(config);
+    //Benchmark(new DenseMatrixProduct(10,100), 100, "10 - 100x100 iterations");
+    //Benchmark(new DenseMatrixProduct(25, 100), 100, "25 - 100x100 iterations");
+    //Benchmark(new DenseMatrixProduct(50, 10), 100, "50 - 100x10 iterations");
+    //Benchmark(new DenseMatrixProduct(100, 10), 100, "100 - 100x10 iterations");
+    //Benchmark(new DenseMatrixProduct(250, 1), 10, "250 - 10x1 iterations");
+    //Benchmark(new DenseMatrixProduct(500,1), 10, "500 - 10x1 iterations");
+    //Benchmark(new DenseMatrixProduct(1000,1), 2, "1000 - 2x1 iterations");
+
     public class DenseMatrixProduct
     {
         readonly Dictionary<string, Matrix<double>> _data = new Dictionary<string, Matrix<double>>();
 
-        readonly ILinearAlgebraProvider _mathnetMkl;
-        readonly ILinearAlgebraProvider _mathnetManaged;
-        //readonly ILinearAlgebraProvider _mathnetExperimental;
-
         [Params(8, 64, 128)]
         public int M { get; set; }
 
         [Params(8, 64, 128)]
         public int N { get; set; }
 
-        static string Key(int m, int n)
+        [Params(Provider.Managed, Provider.NativeMKLAutoHigh, Provider.NativeMKLAvx2High)]
-        {
+        public Provider Provider { get; set; }
-            return $"{m}x{n}";
-        }
 
         public DenseMatrixProduct()
         {
-            foreach (var m in new[] {8, 64, 128})
+            foreach (var m in new[] { 8, 64, 128 })
-            foreach (var n in new[] {8, 64, 128})
+            foreach (var n in new[] { 8, 64, 128 })
             {
                 var key = Key(m, n);
                 _data[key] = Matrix<double>.Build.Random(m, n);
             }
-
-            Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
-            _mathnetMkl = LinearAlgebraControl.CreateNativeMKL();
-            _mathnetManaged = LinearAlgebraControl.CreateManaged();
-            //_mathnetExperimental = new ManagedLinearAlgebraProvider(Variation.Experimental);
-
-            _mathnetMkl.InitializeVerify();
-            _mathnetManaged.InitializeVerify();
-            //_mathnetExperimental.InitializeVerify();
         }
 
-        public void Verify()
+        static string Key(int m, int n)
         {
-            M = 8;
+            return $"{m}x{n}";
-            N = 8;
-            var resultMkl = MathNetMKL().ToRowArrays();
-            var resultManaged = MathNetManaged().ToRowArrays();
-            //var resultExperimental = MathNetExperimental().ToRowArrays();
-            for (int i = 0; i < 8; i++)
-            {
-                for (int j = 0; j < 8; j++)
-                {
-                    if (!resultMkl[i][j].AlmostEqual(resultManaged[i][j], 1e-14))
-                    {
-                        throw new Exception($"Managed [{i}][{j}] {resultManaged[i][j]} != {resultMkl[i][j]}");
-                    }
-                    //if (!resultMkl[i][j].AlmostEqual(resultExperimental[i][j], 1e-14))
-                    //{
-                    //    throw new Exception($"Experimental [{i}][{j}] {resultExperimental[i][j]} != {resultMkl[i][j]}");
-                    //}
-                }
-            }
         }
 
-        [Benchmark(OperationsPerInvoke = 1)]
+        [GlobalSetup]
-        public Matrix<double> MathNetMKL()
+        public void Setup()
         {
-            LinearAlgebraControl.Provider = _mathnetMkl;
+            Providers.ForceProvider(Provider);
-            if (M != N)
-            {
-                return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
-            }
-
-            return _data[Key(M, N)]*_data[Key(M, N)];
         }
 
         [Benchmark(OperationsPerInvoke = 1)]
-        public Matrix<double> MathNetManaged()
+        public Matrix<double> Product()
         {
-            LinearAlgebraControl.Provider = _mathnetManaged;
             if (M != N)
             {
                 return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
             }
 
-            return _data[Key(M, N)] * _data[Key(M, N)];
+            return _data[Key(M, N)]*_data[Key(M, N)];
         }
-
-        //[Benchmark(OperationsPerInvoke = 1)]
-        //public Matrix<double> MathNetExperimental()
-        //{
-        //    LinearAlgebraControl.Provider = _mathnetExperimental;
-        //    if (M != N)
-        //    {
-        //        return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
-        //    }
-
-        //    return _data[Key(M, N)] * _data[Key(M, N)];
-        //}
     }
 }

src/Benchmark/LinearAlgebra/DenseVectorAdd.cs

@@ -8,6 +8,13 @@ using MathNet.Numerics.Threading;
 
 namespace Benchmark.LinearAlgebra
 {
+
+
+//Benchmark(new LinearAlgebra.DenseVectorAdd(10000000,1), 10, "Large (10'000'000) - 10x1 iterations");
+//Benchmark(new LinearAlgebra.DenseVectorAdd(100,1000), 100, "Small (100) - 100x1000 iterations");
+
+
+
 //    public class DenseVectorAdd
 //    {
 //        readonly int _rounds;

src/Benchmark/Program.cs

@@ -1,42 +1,31 @@
 using System;
-using BenchmarkDotNet.Configs;
+using System.Linq;
-using BenchmarkDotNet.Diagnostics.Windows;
-using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Running;
 using MathNet.Numerics;
+using MathNet.Numerics.Providers.FourierTransform;
+using MathNet.Numerics.Providers.LinearAlgebra;
 
 namespace Benchmark
 {
     public class Program
     {
-        public static void Main()
+        public static void Main(string[] args)
         {
-            Providers.ForceNativeMKL();
             Console.WriteLine(Control.Describe());
+            foreach (var provider in Enum.GetValues(typeof(Provider)).Cast<Provider>())
+            {
+                Providers.ForceProvider(provider);
+                Console.WriteLine($"{provider}: LA={LinearAlgebraControl.Provider}, FFT={FourierTransformControl.Provider}");
+            }
 
-            var subject = new LinearAlgebra.DenseMatrixProduct();
+            var switcher = new BenchmarkSwitcher(
-            subject.Verify();
+                new[]
-            Console.WriteLine("Verified.");
+                    {
+                        typeof(Transforms.FFT),
+                        typeof(LinearAlgebra.DenseMatrixProduct),
+                    });
 
-            var config = ManualConfig.Create(DefaultConfig.Instance);
+            switcher.Run(args);
-            config.Add(Job.RyuJitX64, Job.LegacyJitX86);
-            //config.Add(new MemoryDiagnoser());
-
-            BenchmarkRunner.Run<Transforms.FFT>(config);
-            //BenchmarkRunner.Run<LinearAlgebra.DenseMatrixProduct>(config);
-
-            //Benchmark(new LinearAlgebra.DenseVectorAdd(10000000,1), 10, "Large (10'000'000) - 10x1 iterations");
-            //Benchmark(new LinearAlgebra.DenseVectorAdd(100,1000), 100, "Small (100) - 100x1000 iterations");
-
-            //DenseMatrixProduct.Verify(5);
-            //DenseMatrixProduct.Verify(100);
-            //Benchmark(new DenseMatrixProduct(10,100), 100, "10 - 100x100 iterations");
-            //Benchmark(new DenseMatrixProduct(25, 100), 100, "25 - 100x100 iterations");
-            //Benchmark(new DenseMatrixProduct(50, 10), 100, "50 - 100x10 iterations");
-            //Benchmark(new DenseMatrixProduct(100, 10), 100, "100 - 100x10 iterations");
-            //Benchmark(new DenseMatrixProduct(250, 1), 10, "250 - 10x1 iterations");
-            //Benchmark(new DenseMatrixProduct(500,1), 10, "500 - 10x1 iterations");
-            //Benchmark(new DenseMatrixProduct(1000,1), 2, "1000 - 2x1 iterations");
         }
     }
 }

src/Benchmark/Providers.cs

@@ -1,21 +1,45 @@
 using MathNet.Numerics;
+using MathNet.Numerics.Providers.Common.Mkl;
 
 namespace Benchmark
 {
+    public enum Provider : int
+    {
+        Managed = 0,
+        NativeMKLAutoHigh = 1,
+        NativeMKLAutoLow = 2,
+        NativeMKLAvx2High = 3,
+        NativeMKLAvx2Low = 4,
+        NativeOpenBLAS = 5
+    }
+
     public static class Providers
     {
-        public static void ForceNativeMKL()
+        public static void ForceProvider(Provider provider)
         {
-            //Control.NativeProviderPath = @"C:\Triage\NATIVE-Win\";
+            //Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
-            Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
-            Control.UseNativeMKL();
-        }
 
-        public static void ForceOpenBLAS()
+            switch (provider)
-        {
+            {
-            //Control.NativeProviderPath = @"C:\Triage\NATIVE-Win\";
+                case Provider.Managed:
-            Control.NativeProviderPath = @"..\..\..\..\out\OpenBLAS\Windows\";
+                    Control.UseManaged();
-            Control.UseNativeOpenBLAS();
+                    break;
+                case Provider.NativeMKLAutoHigh:
+                    Control.UseNativeMKL(MklConsistency.Auto, MklPrecision.Double, MklAccuracy.High);
+                    break;
+                case Provider.NativeMKLAutoLow:
+                    Control.UseNativeMKL(MklConsistency.Auto, MklPrecision.Double, MklAccuracy.Low);
+                    break;
+                case Provider.NativeMKLAvx2High:
+                    Control.UseNativeMKL(MklConsistency.AVX2, MklPrecision.Double, MklAccuracy.High);
+                    break;
+                case Provider.NativeMKLAvx2Low:
+                    Control.UseNativeMKL(MklConsistency.AVX2, MklPrecision.Double, MklAccuracy.Low);
+                    break;
+                case Provider.NativeOpenBLAS:
+                    Control.UseNativeOpenBLAS();
+                    break;
+            }
         }
     }
 }

src/Benchmark/Transforms/FFT.cs

@@ -1,51 +1,62 @@
-using System;
+using System.Numerics;
-using System.Collections.Generic;
-using System.Numerics;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
 using MathNet.Numerics;
 using MathNet.Numerics.IntegralTransforms;
 
 namespace Benchmark.Transforms
 {
+    [Config(typeof(Config))]
     public class FFT
     {
-        readonly Dictionary<int, Complex[]> _data = new Dictionary<int, Complex[]>();
+        class Config : ManualConfig
-
-        [Params(32, 64, 128, 1024, 8192, 65536)]
-        public int N { get; set; }
-
-        public FFT()
         {
-            var realSinusoidal = Generate.Sinusoidal(65536, 32, -2.0, 2.0);
+            public Config()
-            var imagSawtooth = Generate.Sawtooth(65536, 32, -20.0, 20.0);
-            var signal = Generate.Map2(realSinusoidal, imagSawtooth, (r, i) => new Complex(r, i));
-            foreach (var n in new[] { 32, 64, 128, 1024, 8192, 65536 })
             {
-                var s = new Complex[n];
+                Add(
-                Array.Copy(signal, 0, s, 0, n);
+                    new Job("CLR RyuJit x64", RunMode.Default, EnvMode.RyuJitX64)
-                _data[n] = s;
+                    {
+                        Env = { Runtime = Runtime.Clr, Platform = Platform.X64 }
+                    },
+                    new Job("CLR RyuJit x86", RunMode.Default, EnvMode.RyuJitX86)
+                    {
+                        Env = { Runtime = Runtime.Clr, Platform = Platform.X86 }
+                    },
+                    new Job("CLR LegacyJit x64", RunMode.Default, EnvMode.LegacyJitX64)
+                    {
+                        Env = { Runtime = Runtime.Clr, Platform = Platform.X64 }
+                    },
+                    new Job("CLR LegacyJit x86", RunMode.Default, EnvMode.LegacyJitX86)
+                    {
+                        Env = { Runtime = Runtime.Clr, Platform = Platform.X86 }
+                    });
             }
-
-            Providers.ForceNativeMKL();
         }
 
-        [Setup]
+        [Params(32, 128)] //, 64, 1024, 8192, 65536)]
-        public void Setup()
+        public int N { get; set; }
-        {
+
-        }
+        [Params(Provider.Managed, Provider.NativeMKLAutoHigh, Provider.NativeMKLAvx2High)]
+        public Provider Provider { get; set; }
 
-        [Benchmark(Baseline = true, OperationsPerInvoke = 2)]
+        Complex[] _data;
-        public void Managed()
+
+        [GlobalSetup]
+        public void Setup()
         {
-            Fourier.Radix2Forward(_data[N], FourierOptions.NoScaling);
+            Providers.ForceProvider(Provider);
-            Fourier.Radix2Inverse(_data[N], FourierOptions.NoScaling);
+            var realSinusoidal = Generate.Sinusoidal(N, 32, -2.0, 2.0);
+            var imagSawtooth = Generate.Sawtooth(N, 32, -20.0, 20.0);
+            _data = Generate.Map2(realSinusoidal, imagSawtooth, (r, i) => new Complex(r, i));
         }
 
         [Benchmark(OperationsPerInvoke = 2)]
-        public void NativeMKL()
+        public void Transform()
         {
-            Fourier.Forward(_data[N], FourierOptions.NoScaling);
+            Fourier.Forward(_data, FourierOptions.NoScaling);
-            Fourier.Inverse(_data[N], FourierOptions.NoScaling);
+            Fourier.Inverse(_data, FourierOptions.NoScaling);
         }
     }
 }

src/Benchmark/paket.references

@@ -1,3 +1,2 @@
 group Benchmark
     BenchmarkDotNet
-    BenchmarkDotNet.Diagnostics.Windows

src/Numerics.Tests/Numerics.Tests.CUDA.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
-    <RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
+    <RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <AssemblyName>MathNet.Numerics.Tests.CUDA</AssemblyName>
     <RootNamespace>MathNet.Numerics.Tests</RootNamespace>

src/Numerics.Tests/Numerics.Tests.MKL.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
-    <RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
+    <RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <AssemblyName>MathNet.Numerics.Tests.MKL</AssemblyName>
     <RootNamespace>MathNet.Numerics.Tests</RootNamespace>

src/Numerics.Tests/Numerics.Tests.OpenBLAS.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
-    <RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
+    <RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <AssemblyName>MathNet.Numerics.Tests.OpenBLAS</AssemblyName>
     <RootNamespace>MathNet.Numerics.Tests</RootNamespace>

src/Numerics.Tests/Numerics.Tests.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net45;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>
-    <RuntimeFrameworkVersion>2.0.0</RuntimeFrameworkVersion>
+    <RuntimeFrameworkVersion>2.0.5</RuntimeFrameworkVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <AssemblyName>MathNet.Numerics.Tests</AssemblyName>
     <RootNamespace>MathNet.Numerics.Tests</RootNamespace>