Benchmark: add back to solution, migrate to Paket

9 years ago · 41ce5dc1aa
8 changed files with 168 additions and 40 deletions
--- a/MathNet.Numerics.NativeProviders.sln
+++ b/MathNet.Numerics.NativeProviders.sln
@ -57,7 +57,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{97C9717A
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Numerics.Tests", "src\Numerics.Tests\Numerics.Tests.csproj", "{DAF07AA8-C5C9-4963-98F7-2C3285064DAD}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "src\Benchmark\Benchmark.csproj", "{F2CA84AE-4B7C-46F5-9889-82BC5F9F0F4E}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Benchmark", "src\Benchmark\Benchmark.csproj", "{F2CA84AE-4B7C-46F5-9889-82BC5F9F0F4E}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -210,7 +210,9 @@ Global
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release|Win32.ActiveCfg = Release|Any CPU
+		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release|Win32.Build.0 = Release|Any CPU
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release|x64.ActiveCfg = Release|Any CPU
+		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release|x64.Build.0 = Release|Any CPU
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release-CUDA|Any CPU.ActiveCfg = Release|Any CPU
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release-CUDA|Any CPU.Build.0 = Release|Any CPU
 		{3515A344-AB5F-41C7-A14C-04A79B3FFAB1}.Release-CUDA|Mixed Platforms.ActiveCfg = Release|Any CPU
@ -285,7 +287,9 @@ Global
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release|Win32.ActiveCfg = Release|Any CPU
+		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release|Win32.Build.0 = Release|Any CPU
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release|x64.ActiveCfg = Release|Any CPU
+		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release|x64.Build.0 = Release|Any CPU
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release-CUDA|Any CPU.ActiveCfg = Release|Any CPU
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release-CUDA|Any CPU.Build.0 = Release|Any CPU
 		{E79C0395-01DC-4BC9-B86C-ED45790892C5}.Release-CUDA|Mixed Platforms.ActiveCfg = Release|Any CPU
@ -358,7 +362,9 @@ Global
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release|Win32.ActiveCfg = Release|Any CPU
+		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release|Win32.Build.0 = Release|Any CPU
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release|x64.ActiveCfg = Release|Any CPU
+		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release|x64.Build.0 = Release|Any CPU
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release-CUDA|Any CPU.ActiveCfg = Release|Any CPU
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release-CUDA|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{96B903EF-3EE1-4569-803C-0482D2F5ED37}.Release-CUDA|Win32.ActiveCfg = Release|Any CPU
@ -390,7 +396,9 @@ Global
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release|Win32.ActiveCfg = Release|Any CPU
+		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release|Win32.Build.0 = Release|Any CPU
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release|x64.ActiveCfg = Release|Any CPU
+		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release|x64.Build.0 = Release|Any CPU
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release-CUDA|Any CPU.ActiveCfg = Release|Any CPU
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release-CUDA|Any CPU.Build.0 = Release|Any CPU
 		{AF3253C9-4DB5-45A0-98CF-C105FDA9DA47}.Release-CUDA|Mixed Platforms.ActiveCfg = Release|Any CPU
--- a/MathNet.Numerics.sln
+++ b/MathNet.Numerics.sln
@ -41,6 +41,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Numerics.Tests.OpenBLAS", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Numerics.Tests.CUDA", "src\Numerics.Tests\Numerics.Tests.CUDA.csproj", "{E184F219-F9F9-4396-AE0A-B431375CF06F}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Benchmark", "src\Benchmark\Benchmark.csproj", "{82444930-9FD7-41B6-B476-CCE41A0C0A0B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@ -96,6 +98,12 @@ Global
 		{E184F219-F9F9-4396-AE0A-B431375CF06F}.Release|Any CPU.Build.0 = Release|Any CPU
 		{E184F219-F9F9-4396-AE0A-B431375CF06F}.Release-Signed|Any CPU.ActiveCfg = Debug|Any CPU
 		{E184F219-F9F9-4396-AE0A-B431375CF06F}.Release-Signed|Any CPU.Build.0 = Debug|Any CPU
+		{82444930-9FD7-41B6-B476-CCE41A0C0A0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{82444930-9FD7-41B6-B476-CCE41A0C0A0B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{82444930-9FD7-41B6-B476-CCE41A0C0A0B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{82444930-9FD7-41B6-B476-CCE41A0C0A0B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{82444930-9FD7-41B6-B476-CCE41A0C0A0B}.Release-Signed|Any CPU.ActiveCfg = Release|Any CPU
+		{82444930-9FD7-41B6-B476-CCE41A0C0A0B}.Release-Signed|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/paket.dependencies
+++ b/paket.dependencies
@ -66,6 +66,6 @@ group Data

 group Benchmark
    source https://www.nuget.org/api/v2
-    framework: net45
+    framework: net46
    nuget BenchmarkDotNet content:none, strategy:min
    nuget BenchmarkDotNet.Diagnostics.Windows content:none, strategy:min
--- a/paket.lock
+++ b/paket.lock
@ -1036,11 +1036,121 @@ NUGET
      System.Xml.XPath (>= 4.3) - restriction: || (&& (< monotouch) (>= netstandard1.3) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (>= net46)

 GROUP Benchmark
-RESTRICTION: == net45
+RESTRICTION: == net46
 NUGET
  remote: https://www.nuget.org/api/v2
-    BenchmarkDotNet (0.10.9) - content: none
-    BenchmarkDotNet.Diagnostics.Windows (0.10.9) - content: none
+    BenchmarkDotNet (0.10.12) - content: none
+      BenchmarkDotNet.Core (>= 0.10.12)
+      BenchmarkDotNet.Toolchains.Roslyn (>= 0.10.12)
+    BenchmarkDotNet.Core (0.10.12) - content: none
+      Microsoft.DotNet.InternalAbstractions (>= 1.0)
+      Microsoft.DotNet.PlatformAbstractions (>= 1.1.1)
+      Microsoft.Win32.Registry (>= 4.3)
+      System.Threading.Tasks.Extensions (>= 4.3)
+      System.ValueTuple (>= 4.3)
+      System.Xml.XmlSerializer (>= 4.3)
+    BenchmarkDotNet.Diagnostics.Windows (0.10.12) - content: none
+      BenchmarkDotNet (>= 0.10.12)
+      Microsoft.Diagnostics.Tracing.TraceEvent (>= 1.0.41)
+    BenchmarkDotNet.Toolchains.Roslyn (0.10.12) - content: none
+      BenchmarkDotNet.Core (>= 0.10.12)
+      Microsoft.CodeAnalysis.CSharp (>= 2.4)
+      System.Threading.Tasks (>= 4.3)
+    Microsoft.CodeAnalysis.Analyzers (1.1) - content: none
+    Microsoft.CodeAnalysis.Common (2.4) - content: none
+      Microsoft.CodeAnalysis.Analyzers (>= 1.1)
+      System.AppContext (>= 4.3)
+      System.Collections (>= 4.3)
+      System.Collections.Concurrent (>= 4.3)
+      System.Collections.Immutable (>= 1.3.1)
+      System.Console (>= 4.3)
+      System.Diagnostics.Debug (>= 4.3)
+      System.Diagnostics.FileVersionInfo (>= 4.3)
+      System.Diagnostics.StackTrace (>= 4.3)
+      System.Diagnostics.Tools (>= 4.3)
+      System.Dynamic.Runtime (>= 4.3)
+      System.Globalization (>= 4.3)
+      System.IO.Compression (>= 4.3)
+      System.IO.FileSystem (>= 4.3)
+      System.IO.FileSystem.Primitives (>= 4.3)
+      System.Linq (>= 4.3)
+      System.Linq.Expressions (>= 4.3)
+      System.Reflection (>= 4.3)
+      System.Reflection.Metadata (>= 1.4.2)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
+      System.Runtime.Numerics (>= 4.3)
+      System.Security.Cryptography.Algorithms (>= 4.3)
+      System.Security.Cryptography.Encoding (>= 4.3)
+      System.Security.Cryptography.X509Certificates (>= 4.3)
+      System.Text.Encoding (>= 4.3)
+      System.Text.Encoding.CodePages (>= 4.3)
+      System.Text.Encoding.Extensions (>= 4.3)
+      System.Threading (>= 4.3)
+      System.Threading.Tasks (>= 4.3)
+      System.Threading.Tasks.Parallel (>= 4.3)
+      System.Threading.Thread (>= 4.3)
+      System.ValueTuple (>= 4.3)
+      System.Xml.ReaderWriter (>= 4.3)
+      System.Xml.XDocument (>= 4.3)
+      System.Xml.XmlDocument (>= 4.3)
+      System.Xml.XPath.XDocument (>= 4.3)
+    Microsoft.CodeAnalysis.CSharp (2.4) - content: none
+      Microsoft.CodeAnalysis.Common (2.4)
+    Microsoft.Diagnostics.Tracing.TraceEvent (1.0.41) - content: none
+    Microsoft.DotNet.InternalAbstractions (1.0) - content: none
+    Microsoft.DotNet.PlatformAbstractions (1.1.1) - content: none
+    Microsoft.Win32.Registry (4.3) - content: none
+    System.AppContext (4.3) - content: none
+    System.Collections (4.3) - content: none
+    System.Collections.Concurrent (4.3) - content: none
+    System.Collections.Immutable (1.3.1) - content: none
+    System.Console (4.3) - content: none
+    System.Diagnostics.Debug (4.3) - content: none
+    System.Diagnostics.FileVersionInfo (4.3) - content: none
+    System.Diagnostics.StackTrace (4.3) - content: none
+    System.Diagnostics.Tools (4.3) - content: none
+    System.Dynamic.Runtime (4.3) - content: none
+    System.Globalization (4.3) - content: none
+    System.IO.Compression (4.3) - content: none
+    System.IO.FileSystem (4.3) - content: none
+      System.IO.FileSystem.Primitives (>= 4.3)
+    System.IO.FileSystem.Primitives (4.3) - content: none
+    System.Linq (4.3) - content: none
+    System.Linq.Expressions (4.3) - content: none
+    System.Reflection (4.3) - content: none
+    System.Reflection.Metadata (1.4.2) - content: none
+      System.Collections.Immutable (>= 1.3.1)
+    System.Resources.ResourceManager (4.3) - content: none
+    System.Runtime (4.3) - content: none
+    System.Runtime.Extensions (4.3) - content: none
+    System.Runtime.InteropServices (4.3) - content: none
+    System.Runtime.Numerics (4.3) - content: none
+    System.Security.Cryptography.Algorithms (4.3) - content: none
+      System.Security.Cryptography.Primitives (>= 4.3)
+    System.Security.Cryptography.Encoding (4.3) - content: none
+    System.Security.Cryptography.Primitives (4.3) - content: none
+    System.Security.Cryptography.X509Certificates (4.3) - content: none
+      System.Security.Cryptography.Algorithms (>= 4.3)
+      System.Security.Cryptography.Encoding (>= 4.3)
+    System.Text.Encoding (4.3) - content: none
+    System.Text.Encoding.CodePages (4.3) - content: none
+    System.Text.Encoding.Extensions (4.3) - content: none
+    System.Threading (4.3) - content: none
+    System.Threading.Tasks (4.3) - content: none
+    System.Threading.Tasks.Extensions (4.3) - content: none
+    System.Threading.Tasks.Parallel (4.3) - content: none
+    System.Threading.Thread (4.3) - content: none
+    System.ValueTuple (4.3) - content: none
+    System.Xml.ReaderWriter (4.3) - content: none
+    System.Xml.XDocument (4.3) - content: none
+    System.Xml.XmlDocument (4.3) - content: none
+    System.Xml.XmlSerializer (4.3) - content: none
+    System.Xml.XPath (4.3) - content: none
+    System.Xml.XPath.XDocument (4.3) - content: none
+      System.Xml.XPath (>= 4.3)

 GROUP Build
 NUGET
--- a/src/Benchmark/Benchmark.csproj
+++ b/src/Benchmark/Benchmark.csproj
@ -1,12 +1,12 @@
-<Project Sdk="Microsoft.NET.Sdk">
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <!--<TargetFrameworks>net46;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>-->
-        <TargetFrameworks>net46</TargetFrameworks>
-    </PropertyGroup>
-    <ItemGroup>
-      <PackageReference Include="BenchmarkDotNet" Version="*" />
-      <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="*" Condition="'$(TargetFramework)'=='net46'" />
-      <ProjectReference Include="..\Numerics\Numerics.csproj" />
-    </ItemGroup>
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <!--<TargetFrameworks>net46;netcoreapp1.1;netcoreapp2.0</TargetFrameworks>-->
+    <TargetFrameworks>net46</TargetFrameworks>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Numerics\Numerics.csproj" />
+  </ItemGroup>
+  <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>
--- a/src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs
+++ b/src/Benchmark/LinearAlgebra/DenseMatrixProduct.cs
@ -15,7 +15,7 @@ namespace Benchmark.LinearAlgebra

        readonly ILinearAlgebraProvider _mathnetMkl;
        readonly ILinearAlgebraProvider _mathnetManaged;
-        readonly ILinearAlgebraProvider _mathnetExperimental;
+        //readonly ILinearAlgebraProvider _mathnetExperimental;

        [Params(8, 64, 128)]
        public int M { get; set; }
@ -38,13 +38,13 @@ namespace Benchmark.LinearAlgebra
            }

            Control.NativeProviderPath = @"..\..\..\..\out\MKL\Windows\";
-            _mathnetMkl = new MklLinearAlgebraProvider();
-            _mathnetManaged = new ManagedLinearAlgebraProvider();
-            _mathnetExperimental = new ManagedLinearAlgebraProvider(Variation.Experimental);
+            _mathnetMkl = LinearAlgebraControl.CreateNativeMKL();
+            _mathnetManaged = LinearAlgebraControl.CreateManaged();
+            //_mathnetExperimental = new ManagedLinearAlgebraProvider(Variation.Experimental);

            _mathnetMkl.InitializeVerify();
            _mathnetManaged.InitializeVerify();
-            _mathnetExperimental.InitializeVerify();
+            //_mathnetExperimental.InitializeVerify();
        }

        public void Verify()
@ -53,7 +53,7 @@ namespace Benchmark.LinearAlgebra
            N = 8;
            var resultMkl = MathNetMKL().ToRowArrays();
            var resultManaged = MathNetManaged().ToRowArrays();
-            var resultExperimental = MathNetExperimental().ToRowArrays();
+            //var resultExperimental = MathNetExperimental().ToRowArrays();
            for (int i = 0; i < 8; i++)
            {
                for (int j = 0; j < 8; j++)
@ -62,10 +62,10 @@ namespace Benchmark.LinearAlgebra
                    {
                        throw new Exception($"Managed [{i}][{j}] {resultManaged[i][j]} != {resultMkl[i][j]}");
                    }
-                    if (!resultMkl[i][j].AlmostEqual(resultExperimental[i][j], 1e-14))
-                    {
-                        throw new Exception($"Experimental [{i}][{j}] {resultExperimental[i][j]} != {resultMkl[i][j]}");
-                    }
+                    //if (!resultMkl[i][j].AlmostEqual(resultExperimental[i][j], 1e-14))
+                    //{
+                    //    throw new Exception($"Experimental [{i}][{j}] {resultExperimental[i][j]} != {resultMkl[i][j]}");
+                    //}
                }
            }
        }
@ -73,7 +73,7 @@ namespace Benchmark.LinearAlgebra
        [Benchmark(OperationsPerInvoke = 1)]
        public Matrix<double> MathNetMKL()
        {
-            Control.LinearAlgebraProvider = _mathnetMkl;
+            LinearAlgebraControl.Provider = _mathnetMkl;
            if (M != N)
            {
                return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
@ -85,7 +85,7 @@ namespace Benchmark.LinearAlgebra
        [Benchmark(OperationsPerInvoke = 1)]
        public Matrix<double> MathNetManaged()
        {
-            Control.LinearAlgebraProvider = _mathnetManaged;
+            LinearAlgebraControl.Provider = _mathnetManaged;
            if (M != N)
            {
                return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
@ -94,16 +94,16 @@ namespace Benchmark.LinearAlgebra
            return _data[Key(M, N)] * _data[Key(M, N)];
        }

-        [Benchmark(OperationsPerInvoke = 1)]
-        public Matrix<double> MathNetExperimental()
-        {
-            Control.LinearAlgebraProvider = _mathnetExperimental;
-            if (M != N)
-            {
-                return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
-            }
+        //[Benchmark(OperationsPerInvoke = 1)]
+        //public Matrix<double> MathNetExperimental()
+        //{
+        //    LinearAlgebraControl.Provider = _mathnetExperimental;
+        //    if (M != N)
+        //    {
+        //        return _data[Key(M, N)].TransposeAndMultiply(_data[Key(M, N)]);
+        //    }

-            return _data[Key(M, N)] * _data[Key(M, N)];
-        }
+        //    return _data[Key(M, N)] * _data[Key(M, N)];
+        //}
    }
 }
--- a/src/Benchmark/Program.cs
+++ b/src/Benchmark/Program.cs
@ -12,8 +12,7 @@ namespace Benchmark
        public static void Main()
        {
            Providers.ForceNativeMKL();
-            Console.WriteLine("Linear Algebra:  " + Control.LinearAlgebraProvider);
-            Console.WriteLine("FFT:             " + Control.FourierTransformProvider);
+            Console.WriteLine(Control.Describe());

            var subject = new LinearAlgebra.DenseMatrixProduct();
            subject.Verify();
--- a/src/Benchmark/paket.references
+++ b/src/Benchmark/paket.references
@ -0,0 +1,3 @@
+group Benchmark
+    BenchmarkDotNet
+    BenchmarkDotNet.Diagnostics.Windows