First commit working on optimization tools

src/Numerics/Numerics.csproj

@@ -121,7 +121,6 @@
     <Compile Include="Optimization\Exceptions.cs" />
     <Compile Include="Optimization\ObjectiveFunctions\ObjectiveFunctionBase.cs" />
     <Compile Include="Optimization\ObjectiveFunctions\LazyObjectiveFunction.cs" />
-    <Compile Include="Optimization\ObjectiveFunction.cs" />
     <Compile Include="Optimization\ObjectiveFunctions\ValueObjectiveFunction.cs" />
     <Compile Include="Optimization\ObjectiveFunctions\HessianObjectiveFunction.cs" />
     <Compile Include="Optimization\ObjectiveFunctions\GradientObjectiveFunction.cs" />
@@ -263,6 +262,12 @@
     <Compile Include="RootFinding\Brent.cs" />
     <Compile Include="FindRoots.cs" />
     <Compile Include="RootFinding\Bisection.cs" />
+    <Compile Include="Optimization\BFGS.cs" />
+    <Compile Include="Optimization\ConjugateGradientMinimizer.cs" />
+    <Compile Include="Optimization\GoldenSectionMinimizer.cs" />
+    <Compile Include="Optimization\ObjectiveFunction.cs" />
+    <Compile Include="Optimization\OptimizationResult.cs" />
+    <Compile Include="Optimization\LineSearch\StrongWolfeLineSearch.cs" />
     <Compile Include="SpecialFunctions\Evaluate.cs" />
     <Compile Include="ExcelFunctions.cs" />
     <Compile Include="SpecialFunctions\ExponentialIntegral.cs" />

src/Numerics/Optimization/BFGS.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace MathNet.Numerics.Optimization
+{
+    class BFGS
+    {
+    }
+}

src/Numerics/Optimization/ConjugateGradientMinimizer.cs

@@ -0,0 +1,78 @@
+using System;
+using MathNet.Numerics.LinearAlgebra;
+using MathNet.Numerics.Optimization.LineSearch;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class ConjugateGradientMinimizer
+    {
+        public double GradientTolerance { get; set; }
+        public int MaximumIterations { get; set; }
+
+        public ConjugateGradientMinimizer(double gradientTolerance, int maximumIterations)
+        {
+            this.GradientTolerance = gradientTolerance;
+            this.MaximumIterations = maximumIterations;
+        }
+
+        public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector<double> initialGuess)
+        {
+            if (!objective.IsGradientSupported)
+                throw new Exception("Gradient not supported in objective function, but required for ConjugateGradient minimization.");
+
+            objective.EvaluateAt(initialGuess);
+            var gradient = objective.Gradient;
+            ValidateGradient(gradient, initialGuess);
+
+            // Check that we're not already done
+            if (ExitCriteriaSatisfied(initialGuess, gradient))
+                return new MinimizationResult(objective, 0, MinimizationResult.ExitCondition.AbsoluteGradient);
+
+            // Set up line search algorithm
+            var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, 1000);
+
+            // First step
+            var steepestDirection = -gradient;
+            var searchDirection = steepestDirection;
+            var result = lineSearcher.FindConformingStep(objective, searchDirection, 1.0);
+            objective = result.FunctionInfoAtMinimum;
+            ValidateGradient(objective.Gradient, objective.Point);
+
+            // Subsequent steps
+            int iterations = 1;
+            while (!ExitCriteriaSatisfied(objective.Point, objective.Gradient) && iterations < MaximumIterations)
+            {
+                var previousSteepestDirection = steepestDirection;
+                steepestDirection = -objective.Gradient;
+                var searchDirectionAdjuster = steepestDirection * (steepestDirection - previousSteepestDirection) / (previousSteepestDirection * previousSteepestDirection);
+                searchDirection = steepestDirection + searchDirectionAdjuster * previousSteepestDirection;
+                result = lineSearcher.FindConformingStep(objective, searchDirection, 1.0);
+
+                objective = result.FunctionInfoAtMinimum;
+                iterations += 1;
+            }
+
+            return new MinimizationResult(objective, iterations, MinimizationResult.ExitCondition.AbsoluteGradient);
+        }
+
+        private bool ExitCriteriaSatisfied(Vector<double> candidatePoint, Vector<double> gradient)
+        {
+            return gradient.Norm(2.0) < GradientTolerance;
+        }
+
+        private void ValidateGradient(Vector<double> gradient, Vector<double> input)
+        {
+            foreach (var x in gradient)
+            {
+                if (Double.IsNaN(x) || Double.IsInfinity(x))
+                    throw new Exception("Non-finite gradient returned.");
+            }
+        }
+
+        private void ValidateObjective(double objective, Vector<double> input)
+        {
+            if (Double.IsNaN(objective) || Double.IsInfinity(objective))
+                throw new Exception("Non-finite objective function returned.");
+        }
+    }
+}

src/Numerics/Optimization/GoldenSectionMinimizer.cs

@@ -0,0 +1,6 @@
+namespace MathNet.Numerics.Optimization
+{
+    class GoldenSectionMinimizer
+    {
+    }
+}

src/Numerics/Optimization/LineSearch/StrongWolfeLineSearch.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace MathNet.Numerics.Optimization
+{
+    class StrongWolfeLineSearch
+    {
+    }
+}

src/Numerics/Optimization/OptimizationResult.cs

@@ -0,0 +1,8 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace MathNet.Numerics.Optimization
+{
+}

src/UnitTests/OptimizationTests/TestConjugateGradientMinimizer.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using NUnit.Framework;
+
+using MathNet.Numerics.Optimization;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    [TestFixture]
+    public class TestConjugateGradientMinimizer
+    {
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Easy()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+            var solver = new ConjugateGradientMinimizer(1e-5, 100);
+            var result = solver.FindMinimum(obj, new MathNet.Numerics.LinearAlgebra.Double.DenseVector(new double[]{1.2,1.2}));
+            Assert.That(result.MinimizingPoint[0], Is.EqualTo(1.0));
+            Assert.That(result.MinimizingPoint[1], Is.EqualTo(1.0));
+        }
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Hard()
+        {
+
+        }
+    }
+}

src/UnitTests/UnitTests.csproj

@@ -371,6 +371,12 @@
     <Compile Include="Random\SystemRandomSourceTests.cs" />
     <Compile Include="OptimizationTests\BfgsTest.cs" />
     <Compile Include="RootFindingTests\BisectionTest.cs" />
+    <Compile Include="MatrixHelpers.cs" />
+    <Compile Include="NumberTheoryTests\GcdRelatedTest.cs" />
+    <Compile Include="NumberTheoryTests\GcdRelatedTestBigInteger.cs" />
+    <Compile Include="NumberTheoryTests\IntegerTheoryTest.cs" />
+    <Compile Include="OptimizationTests\RosenbrockFunction.cs" />
+    <Compile Include="OptimizationTests\TestConjugateGradientMinimizer.cs" />
     <Compile Include="PermutationTest.cs" />
     <Compile Include="PrecisionTest.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -423,6 +429,11 @@
   <ItemGroup>
     <None Include="paket.references" />
   </ItemGroup>
+  <ItemGroup>
+    <Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Choose>
     <When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And ($(TargetFrameworkVersion) == 'v2.0' Or $(TargetFrameworkVersion) == 'v3.0')">