Optimization: Fix bug in ObjectiveChecker and ObjectiveChecker1D

* Bug caused the checks to be run every time a value/gradient/hessian was
requested, not just the first time a request was made.

(updated by cdrnet)

.gitignore

@@ -24,3 +24,5 @@ UpgradeLog.htm
 *.so
 *.sdf
 *.opensdf
+src/UnitTests/test-results
+*.userprefs

src/Numerics/Numerics.csproj

@@ -195,6 +195,21 @@
     <Compile Include="RootFinding\Brent.cs" />
     <Compile Include="FindRoots.cs" />
     <Compile Include="RootFinding\Bisection.cs" />
+    <Compile Include="Optimization\BfgsMinimizer.cs" />
+    <Compile Include="Optimization\BisectionRootFinder.cs" />
+    <Compile Include="Optimization\ConjugateGradientMinimizer.cs" />
+    <Compile Include="Optimization\Exceptions.cs" />
+    <Compile Include="Optimization\GoldenSectionMinimizer.cs" />
+    <Compile Include="Optimization\MinimizationWithLineSearchOutput.cs" />
+    <Compile Include="Optimization\LineSearchOutput.cs" />
+    <Compile Include="Optimization\MinimizationOutput.cs" />
+    <Compile Include="Optimization\MinimizationOutput1D.cs" />
+    <Compile Include="Optimization\NewtonMinimizer.cs" />
+    <Compile Include="Optimization\ObjectiveChecker.cs" />
+    <Compile Include="Optimization\ObjectiveChecker1D.cs" />
+    <Compile Include="Optimization\ObjectiveFunction.cs" />
+    <Compile Include="Optimization\ObjectiveFunction1D.cs" />
+    <Compile Include="Optimization\WeakWolfeLineSearch.cs" />
     <Compile Include="SpecialFunctions\Evaluate.cs" />
     <Compile Include="ExcelFunctions.cs" />
     <Compile Include="SpecialFunctions\ModifiedStruve.cs" />
@@ -430,6 +445,7 @@
     <Compile Include="Threading\CommonParallel.cs" />
     <Compile Include="Trigonometry.cs" />
     <Compile Include="Window.cs" />
+    <Compile Include="Optimization\ExitCondition.cs" />
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="Properties\Resources.resx">

src/Numerics/Optimization/BfgsMinimizer.cs

@@ -0,0 +1,185 @@
+// <copyright file="BfgsMinimizer.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class BfgsMinimizer
+    {
+        public double GradientTolerance { get; set; }
+        public double ParameterTolerance { get; set; }
+        public int MaximumIterations { get; set; }
+
+        public BfgsMinimizer(double gradientTolerance, double parameterTolerance, int maximumIterations = 1000)
+        {
+            GradientTolerance = gradientTolerance;
+            ParameterTolerance = parameterTolerance;
+            MaximumIterations = maximumIterations;
+        }
+
+        public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector<double> initialGuess)
+        {
+            if (!objective.GradientSupported)
+                throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization.");
+
+            if (!(objective is ObjectiveChecker))
+                objective = new ObjectiveChecker(objective, ValidateObjective, ValidateGradient, null);
+
+            IEvaluation initialEval = objective.Evaluate(initialGuess);
+
+            // Check that we're not already done
+            ExitCondition currentExitCondition = ExitCriteriaSatisfied(initialEval, null);
+            if (currentExitCondition != ExitCondition.None)
+                return new MinimizationOutput(initialEval, 0, currentExitCondition);
+
+            // Set up line search algorithm
+            var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, ParameterTolerance, maxIterations: 1000);
+
+            // Declare state variables
+            Vector<double> gradient;
+
+            // First step
+            Matrix<double> inversePseudoHessian = Matrix<double>.Build.DiagonalIdentity(initialGuess.Count);
+            Vector<double> searchDirection = -initialEval.Gradient;
+            double stepSize = 100*GradientTolerance/(searchDirection*searchDirection);
+
+            LineSearchOutput result;
+            try
+            {
+                result = lineSearcher.FindConformingStep(objective, initialEval, searchDirection, stepSize);
+            }
+            catch (Exception e)
+            {
+                throw new InnerOptimizationException("Line search failed.", e);
+            }
+
+            IEvaluation previousPoint = initialEval;
+            IEvaluation candidatePoint = result.FunctionInfoAtMinimum;
+            gradient = candidatePoint.Gradient;
+            Vector<double> step = candidatePoint.Point - initialGuess;
+            stepSize = result.FinalStep;
+
+            // Subsequent steps
+            int iterations;
+            int totalLineSearchSteps = result.Iterations;
+            int iterationsWithNontrivialLineSearch = result.Iterations > 0 ? 0 : 1;
+            for (iterations = 1; iterations < MaximumIterations; ++iterations)
+            {
+                var y = candidatePoint.Gradient - previousPoint.Gradient;
+
+                double sy = step*y;
+                inversePseudoHessian = inversePseudoHessian + ((sy + y*inversePseudoHessian*y)/Math.Pow(sy, 2.0))*step.OuterProduct(step) - ((inversePseudoHessian*y.ToColumnMatrix())*step.ToRowMatrix() + step.ToColumnMatrix()*(y.ToRowMatrix()*inversePseudoHessian))*(1.0/sy);
+
+                searchDirection = -inversePseudoHessian*candidatePoint.Gradient;
+
+                if (searchDirection*candidatePoint.Gradient >= -GradientTolerance*GradientTolerance)
+                {
+                    searchDirection = -candidatePoint.Gradient;
+                    inversePseudoHessian = Matrix<double>.Build.DiagonalIdentity(initialGuess.Count);
+                }
+
+                try
+                {
+                    result = lineSearcher.FindConformingStep(objective, candidatePoint, searchDirection, 1.0);
+                }
+                catch (Exception e)
+                {
+                    throw new InnerOptimizationException("Line search failed.", e);
+                }
+
+                iterationsWithNontrivialLineSearch += result.Iterations > 0 ? 1 : 0;
+                totalLineSearchSteps += result.Iterations;
+
+                stepSize = result.FinalStep;
+                step = result.FunctionInfoAtMinimum.Point - candidatePoint.Point;
+                previousPoint = candidatePoint;
+                candidatePoint = result.FunctionInfoAtMinimum;
+
+                currentExitCondition = ExitCriteriaSatisfied(candidatePoint, previousPoint);
+                if (currentExitCondition != ExitCondition.None)
+                    break;
+            }
+
+            if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None)
+                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
+
+            return new MinimizationWithLineSearchOutput(candidatePoint, iterations, currentExitCondition, totalLineSearchSteps, iterationsWithNontrivialLineSearch);
+        }
+
+        ExitCondition ExitCriteriaSatisfied(IEvaluation candidatePoint, IEvaluation lastPoint)
+        {
+            Vector<double> relGrad = new LinearAlgebra.Double.DenseVector(candidatePoint.Point.Count);
+            double relativeGradient = 0.0;
+            double normalizer = Math.Max(Math.Abs(candidatePoint.Value), 1.0);
+            for (int ii = 0; ii < relGrad.Count; ++ii)
+            {
+                double tmp = candidatePoint.Gradient[ii]*Math.Max(Math.Abs(candidatePoint.Point[ii]), 1.0)/normalizer;
+                relativeGradient = Math.Max(relativeGradient, Math.Abs(tmp));
+            }
+            if (relativeGradient < GradientTolerance)
+            {
+                return ExitCondition.RelativeGradient;
+            }
+
+            if (lastPoint != null)
+            {
+                double mostProgress = 0.0;
+                for (int ii = 0; ii < candidatePoint.Point.Count; ++ii)
+                {
+                    var tmp = Math.Abs(candidatePoint.Point[ii] - lastPoint.Point[ii])/Math.Max(Math.Abs(lastPoint.Point[ii]), 1.0);
+                    mostProgress = Math.Max(mostProgress, tmp);
+                }
+                if (mostProgress < ParameterTolerance)
+                {
+                    return ExitCondition.LackOfProgress;
+                }
+            }
+
+            return ExitCondition.None;
+        }
+
+        void ValidateGradient(IEvaluation eval)
+        {
+            foreach (var x in eval.Gradient)
+            {
+                if (Double.IsNaN(x) || Double.IsInfinity(x))
+                    throw new EvaluationException("Non-finite gradient returned.", eval);
+            }
+        }
+
+        void ValidateObjective(IEvaluation eval)
+        {
+            if (Double.IsNaN(eval.Value) || Double.IsInfinity(eval.Value))
+                throw new EvaluationException("Non-finite objective function returned.", eval);
+        }
+    }
+}

src/Numerics/Optimization/BisectionRootFinder.cs

@@ -0,0 +1,132 @@
+// <copyright file="BisectionRootFinder.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class BisectionRootFinder
+    {
+        public double ObjectiveTolerance { get; set; }
+        public double XTolerance { get; set; }
+        public double LowerExpansionFactor { get; set; }
+        public double UpperExpansionFactor { get; set; }
+        public int MaxExpansionSteps { get; set; }
+        public bool AllowInfiniteObjectiveValues { get; set; }
+        public int MaxBisectionSteps { get; set; }
+
+        public BisectionRootFinder(double objectiveTolerance = 1e-5, double xTolerance = 1e-5, double lowerExpansionFactor = -1.0, double upperExpansionFactor = -1.0, int maxExpansionSteps = 10, bool allowInfiniteObjectiveValues=false, int maxBisectionSteps=1000)
+        {
+            ObjectiveTolerance = objectiveTolerance;
+            XTolerance = xTolerance;
+            LowerExpansionFactor = lowerExpansionFactor;
+            UpperExpansionFactor = upperExpansionFactor;
+            MaxExpansionSteps = maxExpansionSteps;
+            AllowInfiniteObjectiveValues = allowInfiniteObjectiveValues;
+            MaxBisectionSteps = maxBisectionSteps;
+        }
+
+        public double FindRoot(Func<double, double> objectiveFunction, double lowerBound, double upperBound)
+        {
+            double lowerVal = objectiveFunction(lowerBound);
+            double upperVal = objectiveFunction(upperBound);
+
+            if (lowerVal == 0.0)
+                return lowerBound;
+            if (upperVal == 0.0)
+                return upperBound;
+
+            ValidateEvaluation(lowerVal, lowerBound);
+            ValidateEvaluation(upperVal, upperBound);
+
+            if (Math.Sign(lowerVal) == Math.Sign(upperVal) && LowerExpansionFactor <= 1.0 && UpperExpansionFactor <= 1.0)
+                throw new Exception("Bounds do not necessarily span a root, and StepExpansionFactor is not set to expand the interval in this case.");
+
+            int expansionSteps = 0;
+            while (Math.Sign(lowerVal) == Math.Sign(upperVal) && expansionSteps < MaxExpansionSteps)
+            {
+                double midpoint = 0.5*(upperBound + lowerBound);
+                double range = upperBound - lowerBound;
+                if (UpperExpansionFactor <= 0.0 || (LowerExpansionFactor > 0.0 && Math.Abs(lowerVal) < Math.Abs(upperVal)))
+                {
+                    lowerBound = upperBound - LowerExpansionFactor*range;
+                    lowerVal = objectiveFunction(lowerBound);
+                    ValidateEvaluation(lowerVal, lowerBound);
+                }
+                else
+                {
+                    upperBound = lowerBound + UpperExpansionFactor*range;
+                    upperVal = objectiveFunction(upperBound);
+                    ValidateEvaluation(upperVal, upperBound);
+                }
+                expansionSteps += 1;
+            }
+
+            if (Math.Sign(lowerVal) == Math.Sign(upperVal) && expansionSteps == MaxExpansionSteps)
+                throw new MaximumIterationsException("Could not bound root in maximum expansion iterations.");
+
+            int bisectionSteps = 0;
+            while ( (Math.Abs(upperVal - lowerVal) > 0.5 * ObjectiveTolerance || Math.Abs(upperBound - lowerBound) > 0.5 * XTolerance) && bisectionSteps < MaxBisectionSteps)
+            {
+                double midpoint = 0.5*(upperBound + lowerBound);
+                double midval = objectiveFunction(midpoint);
+                ValidateEvaluation(midval, midpoint);
+
+                if (Math.Sign(midval) == Math.Sign(lowerVal))
+                {
+                    lowerBound = midpoint;
+                    lowerVal = midval;
+                }
+                else if (Math.Sign(midval) == Math.Sign(upperVal))
+                {
+                    upperBound = midpoint;
+                    upperVal = midval;
+                }
+                else
+                {
+                    return midpoint;
+                }
+
+                bisectionSteps += 1;
+            }
+
+            if (Math.Abs(upperVal - lowerVal) <= 0.5*ObjectiveTolerance && Math.Abs(upperBound - lowerBound) <= 0.5*XTolerance)
+                return 0.5*(lowerBound + upperBound);
+
+            throw new MaximumIterationsException("Bisection did not find root in interval; function is probably non-monotone or discontinuous.");
+        }
+
+        void ValidateEvaluation(double output, double input)
+        {
+            if (Double.IsNaN(output) || (!AllowInfiniteObjectiveValues && Double.IsInfinity(output)))
+                throw new Exception(String.Format("Objective function returned non-finite result: f({0}) = {1}", input, output));
+        }
+    }
+}

src/Numerics/Optimization/ConjugateGradientMinimizer.cs

@@ -0,0 +1,154 @@
+// <copyright file="ConjugateGradientMinimizer.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class ConjugateGradientMinimizer
+    {
+        public double GradientTolerance { get; set; }
+        public int MaximumIterations { get; set; }
+
+        public ConjugateGradientMinimizer(double gradientTolerance, int maximumIterations)
+        {
+            GradientTolerance = gradientTolerance;
+            MaximumIterations = maximumIterations;
+        }
+
+        public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector<double> initialGuess)
+        {
+            if (!objective.GradientSupported)
+                throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization.");
+
+            if (!(objective is ObjectiveChecker))
+                objective = new ObjectiveChecker(objective, ValidateObjective, ValidateGradient, null);
+
+            IEvaluation initialEval = objective.Evaluate(initialGuess);
+            var gradient = initialEval.Gradient;
+
+            // Check that we're not already done
+            if (ExitCriteriaSatisfied(initialGuess, gradient))
+                return new MinimizationOutput(initialEval, 0, ExitCondition.AbsoluteGradient);
+
+            // Set up line search algorithm
+            var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, maxIterations: 1000);
+
+            // Declare state variables
+
+            // First step
+            Vector<double> steepestDirection = -gradient;
+            Vector<double> searchDirection = steepestDirection;
+            double initialStepSize = 100*GradientTolerance/(gradient*gradient);
+            LineSearchOutput result;
+            try
+            {
+                result = lineSearcher.FindConformingStep(objective, initialEval, searchDirection, initialStepSize);
+            }
+            catch (Exception e)
+            {
+                throw new InnerOptimizationException("Line search failed.", e);
+            }
+
+            IEvaluation candidatePoint = result.FunctionInfoAtMinimum;
+
+            double stepSize = result.FinalStep;
+
+            // Subsequent steps
+            int iterations = 1;
+            int totalLineSearchSteps = result.Iterations;
+            int iterationsWithNontrivialLineSearch = result.Iterations > 0 ? 0 : 1;
+            while (!ExitCriteriaSatisfied(candidatePoint.Point, candidatePoint.Gradient) && iterations < MaximumIterations)
+            {
+                Vector<double> previousSteepestDirection = steepestDirection;
+                steepestDirection = -candidatePoint.Gradient;
+                var searchDirectionAdjuster = Math.Max(0, steepestDirection*(steepestDirection - previousSteepestDirection)/(previousSteepestDirection*previousSteepestDirection));
+
+                //double prev_grad_mag = previous_steepest_direction*previous_steepest_direction;
+                //double grad_overlap = steepest_direction*previous_steepest_direction;
+                //double search_grad_overlap = candidate_point.Gradient*search_direction;
+
+                //if (iterations % initial_guess.Count == 0 || (Math.Abs(grad_overlap) >= 0.2 * prev_grad_mag) || (-2 * prev_grad_mag >= search_grad_overlap) || (search_grad_overlap >= -0.2 * prev_grad_mag))
+                //    search_direction = steepest_direction;
+                //else
+                //    search_direction = steepest_direction + search_direction_adjuster * search_direction;
+
+                searchDirection = steepestDirection + searchDirectionAdjuster*searchDirection;
+                if (searchDirection*candidatePoint.Gradient >= 0)
+                {
+                    searchDirection = steepestDirection;
+                }
+
+                try
+                {
+                    result = lineSearcher.FindConformingStep(objective, candidatePoint, searchDirection, stepSize);
+                }
+                catch (Exception e)
+                {
+                    throw new InnerOptimizationException("Line search failed.", e);
+                }
+
+                iterationsWithNontrivialLineSearch += result.Iterations > 0 ? 1 : 0;
+                totalLineSearchSteps += result.Iterations;
+
+                stepSize = result.FinalStep;
+                candidatePoint = result.FunctionInfoAtMinimum;
+
+                iterations += 1;
+            }
+
+            if (iterations == MaximumIterations)
+                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
+
+            return new MinimizationWithLineSearchOutput(candidatePoint, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch);
+        }
+
+        bool ExitCriteriaSatisfied(Vector<double> candidatePoint, Vector<double> gradient)
+        {
+            return gradient.Norm(2.0) < GradientTolerance;
+        }
+
+        void ValidateGradient(IEvaluation eval)
+        {
+            foreach (var x in eval.Gradient)
+            {
+                if (Double.IsNaN(x) || Double.IsInfinity(x))
+                    throw new EvaluationException("Non-finite gradient returned.", eval);
+            }
+        }
+
+        void ValidateObjective(IEvaluation eval)
+        {
+            if (Double.IsNaN(eval.Value) || Double.IsInfinity(eval.Value))
+                throw new EvaluationException("Non-finite objective function returned.", eval);
+        }
+    }
+}

src/Numerics/Optimization/Exceptions.cs

@@ -0,0 +1,106 @@
+// <copyright file="Exceptions.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class OptimizationException : Exception
+    {
+        public OptimizationException(string message)
+            : base(message)
+        {
+        }
+
+        public OptimizationException(string message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+    }
+
+    public class MaximumIterationsException : OptimizationException
+    {
+        public MaximumIterationsException(string message)
+            : base(message)
+        {
+        }
+    }
+
+    public class EvaluationException : OptimizationException
+    {
+        public IEvaluation Evaluation { get; private set; }
+
+        public EvaluationException(string message, IEvaluation eval)
+            : base(message)
+        {
+            Evaluation = eval;
+        }
+
+        public EvaluationException(string message, IEvaluation eval, Exception innerException)
+            : base(message, innerException)
+        {
+            Evaluation = eval;
+        }
+
+        public EvaluationException(string message, IEvaluation1D eval)
+            : base(message)
+        {
+            Evaluation = new OneDEvaluationExpander(eval);
+        }
+
+        public EvaluationException(string message, IEvaluation1D eval, Exception innerException)
+            : base(message, innerException)
+        {
+            Evaluation = new OneDEvaluationExpander(eval);
+        }
+
+    }
+
+    public class InnerOptimizationException : OptimizationException
+    {
+        public InnerOptimizationException(string message)
+            : base(message)
+        {
+        }
+
+        public InnerOptimizationException(string message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+    }
+
+    public class IncompatibleObjectiveException : OptimizationException
+    {
+        public IncompatibleObjectiveException(string message)
+            : base(message)
+        {
+        }
+    }
+}

src/Numerics/Optimization/ExitCondition.cs

@@ -0,0 +1,42 @@
+// <copyright file="ExitCondition.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+namespace MathNet.Numerics
+{
+    public enum ExitCondition
+    {
+        None,
+        RelativeGradient,
+        LackOfProgress,
+        AbsoluteGradient,
+        WeakWolfeCriteria,
+        BoundTolerance
+    }
+}

src/Numerics/Optimization/GoldenSectionMinimizer.cs

@@ -0,0 +1,110 @@
+// <copyright file="GoldenSectionMinimizer.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class GoldenSectionMinimizer
+    {
+        public double XTolerance { get; set; }
+        public int MaximumIterations { get; set; }
+
+        public GoldenSectionMinimizer(double xTolerance = 1e-5, int maxIterations = 1000)
+        {
+            XTolerance = xTolerance;
+            MaximumIterations = maxIterations;
+        }
+
+        public MinimizationOutput1D FindMinimum(IObjectiveFunction1D objective, double lowerBound, double upperBound)
+        {
+            if (!(objective is ObjectiveChecker1D))
+                objective = new ObjectiveChecker1D(objective, ValueChecker, null, null);
+
+            double middlePointX = lowerBound + (upperBound - lowerBound)/(1 + GoldenRatio);
+            IEvaluation1D lower = objective.Evaluate(lowerBound);
+            IEvaluation1D middle = objective.Evaluate(middlePointX);
+            IEvaluation1D upper = objective.Evaluate(upperBound);
+
+            if (upperBound <= lowerBound)
+                throw new OptimizationException("Lower bound must be lower than upper bound.");
+
+            if (upper.Value < middle.Value || lower.Value < middle.Value)
+                throw new OptimizationException("Lower and upper bounds do not necessarily bound a minimum.");
+
+            int iterations = 0;
+            while (Math.Abs(upper.Point - lower.Point) > XTolerance && iterations < MaximumIterations)
+            {
+                double testX = lower.Point + (upper.Point - middle.Point);
+                var test = objective.Evaluate(testX);
+
+                if (test.Point < middle.Point)
+                {
+                    if (test.Value > middle.Value)
+                    {
+                        lower = test;
+                    }
+                    else
+                    {
+                        upper = middle;
+                        middle = test;
+                    }
+                }
+                else
+                {
+                    if (test.Value > middle.Value)
+                    {
+                        upper = test;
+                    }
+                    else
+                    {
+                        lower = middle;
+                        middle = test;
+                    }
+                }
+
+                iterations += 1;
+            }
+
+            if (iterations == MaximumIterations)
+                throw new MaximumIterationsException("Max iterations reached.");
+
+            return new MinimizationOutput1D(middle, iterations, ExitCondition.BoundTolerance);
+        }
+
+        void ValueChecker(IEvaluation1D eval)
+        {
+            if (Double.IsNaN(eval.Value) || Double.IsInfinity(eval.Value))
+                throw new EvaluationException("Objective function returned non-finite value.", eval);
+        }
+
+        static readonly double GoldenRatio = (1.0 + Math.Sqrt(5))/2.0;
+    }
+}

src/Numerics/Optimization/LineSearchOutput.cs

@@ -0,0 +1,43 @@
+// <copyright file="LineSearchOutput.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+namespace MathNet.Numerics.Optimization
+{
+    public class LineSearchOutput : MinimizationOutput
+    {
+        public double FinalStep { get; private set; }
+
+        public LineSearchOutput(IEvaluation functionInfo, int iterations, double finalStep, ExitCondition reasonForExit)
+            : base(functionInfo, iterations, reasonForExit)
+        {
+            FinalStep = finalStep;
+        }
+    }
+}

src/Numerics/Optimization/MinimizationOutput.cs

@@ -0,0 +1,53 @@
+// <copyright file="MinimizationOutput.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class MinimizationOutput
+    {
+        public Vector<double> MinimizingPoint
+        {
+            get { return FunctionInfoAtMinimum.Point; }
+        }
+
+        public IEvaluation FunctionInfoAtMinimum { get; private set; }
+        public int Iterations { get; private set; }
+        public ExitCondition ReasonForExit { get; private set; }
+
+        public MinimizationOutput(IEvaluation functionInfo, int iterations, ExitCondition reasonForExit)
+        {
+            FunctionInfoAtMinimum = functionInfo;
+            Iterations = iterations;
+            ReasonForExit = reasonForExit;
+        }
+    }
+}

src/Numerics/Optimization/MinimizationOutput1D.cs

@@ -0,0 +1,51 @@
+// <copyright file="MinimizationOutput1D.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+namespace MathNet.Numerics.Optimization
+{
+    public class MinimizationOutput1D
+    {
+        public double MinimizingPoint
+        {
+            get { return FunctionInfoAtMinimum.Point; }
+        }
+
+        public IEvaluation1D FunctionInfoAtMinimum { get; private set; }
+        public int Iterations { get; private set; }
+        public ExitCondition ReasonForExit { get; private set; }
+
+        public MinimizationOutput1D(IEvaluation1D functionInfo, int iterations, ExitCondition reasonForExit)
+        {
+            FunctionInfoAtMinimum = functionInfo;
+            Iterations = iterations;
+            ReasonForExit = reasonForExit;
+        }
+    }
+}

src/Numerics/Optimization/MinimizationWithLineSearchOutput.cs

@@ -0,0 +1,45 @@
+// <copyright file="MinimizationWithLineSearchOutput.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+namespace MathNet.Numerics.Optimization
+{
+    public class MinimizationWithLineSearchOutput : MinimizationOutput
+    {
+        public int TotalLineSearchIterations { get; private set; }
+        public int IterationsWithNonTrivialLineSearch { get; private set; }
+
+        public MinimizationWithLineSearchOutput(IEvaluation functionInfo, int iterations, ExitCondition reasonForExit, int totalLineSearchIterations, int iterationsWithNonTrivialLineSearch)
+            : base(functionInfo, iterations, reasonForExit)
+        {
+            TotalLineSearchIterations = totalLineSearchIterations;
+            IterationsWithNonTrivialLineSearch = iterationsWithNonTrivialLineSearch;
+        }
+    }
+}

src/Numerics/Optimization/NewtonMinimizer.cs

@@ -0,0 +1,151 @@
+// <copyright file="NewtonMinimizer.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class NewtonMinimizer
+    {
+        public double GradientTolerance { get; set; }
+        public int MaximumIterations { get; set; }
+        public bool UseLineSearch { get; set; }
+
+        public NewtonMinimizer(double gradientTolerance, int maximumIterations, bool useLineSearch = false)
+        {
+            GradientTolerance = gradientTolerance;
+            MaximumIterations = maximumIterations;
+            UseLineSearch = useLineSearch;
+        }
+
+        public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector<double> initialGuess)
+        {
+            if (!objective.GradientSupported)
+                throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization.");
+
+            if (!objective.HessianSupported)
+                throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization.");
+
+            if (!(objective is ObjectiveChecker))
+                objective = new ObjectiveChecker(objective, ValidateObjective, ValidateGradient, ValidateHessian);
+
+            IEvaluation initialEval = objective.Evaluate(initialGuess);
+
+            // Check that we're not already done
+            if (ExitCriteriaSatisfied(initialGuess, initialEval.Gradient))
+                return new MinimizationOutput(initialEval, 0, ExitCondition.AbsoluteGradient);
+
+            // Set up line search algorithm
+            var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, maxIterations: 1000);
+
+            // Declare state variables
+            IEvaluation candidatePoint = initialEval;
+
+            // Subsequent steps
+            int iterations = 0;
+            int totalLineSearchSteps = 0;
+            int iterationsWithNontrivialLineSearch = 0;
+            bool tmpLineSearch = false;
+            while (!ExitCriteriaSatisfied(candidatePoint.Point, candidatePoint.Gradient) && iterations < MaximumIterations)
+            {
+
+                Vector<double> searchDirection = candidatePoint.Hessian.LU().Solve(-candidatePoint.Gradient);
+
+                if (searchDirection*candidatePoint.Gradient >= 0)
+                {
+                    searchDirection = -candidatePoint.Gradient;
+                    tmpLineSearch = true;
+                }
+
+                if (UseLineSearch || tmpLineSearch)
+                {
+                    LineSearchOutput result;
+                    try
+                    {
+                        result = lineSearcher.FindConformingStep(objective, candidatePoint, searchDirection, 1.0);
+                    }
+                    catch (Exception e)
+                    {
+                        throw new InnerOptimizationException("Line search failed.", e);
+                    }
+                    iterationsWithNontrivialLineSearch += result.Iterations > 0 ? 1 : 0;
+                    totalLineSearchSteps += result.Iterations;
+                    candidatePoint = result.FunctionInfoAtMinimum;
+                }
+                else
+                {
+                    candidatePoint = objective.Evaluate(candidatePoint.Point + searchDirection);
+                }
+
+                tmpLineSearch = false;
+
+                iterations += 1;
+            }
+
+            if (iterations == MaximumIterations)
+                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
+
+            return new MinimizationWithLineSearchOutput(candidatePoint, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch);
+        }
+
+        bool ExitCriteriaSatisfied(Vector<double> candidatePoint, Vector<double> gradient)
+        {
+            return gradient.Norm(2.0) < GradientTolerance;
+        }
+
+        void ValidateGradient(IEvaluation eval)
+        {
+            foreach (var x in eval.Gradient)
+            {
+                if (Double.IsNaN(x) || Double.IsInfinity(x))
+                    throw new EvaluationException("Non-finite gradient returned.", eval);
+            }
+        }
+
+        void ValidateObjective(IEvaluation eval)
+        {
+            if (Double.IsNaN(eval.Value) || Double.IsInfinity(eval.Value))
+                throw new EvaluationException("Non-finite objective function returned.", eval);
+        }
+
+        void ValidateHessian(IEvaluation eval)
+        {
+            for (int ii = 0; ii < eval.Hessian.RowCount; ++ii)
+            {
+                for (int jj = 0; jj < eval.Hessian.ColumnCount; ++jj)
+                {
+                    if (Double.IsNaN(eval.Hessian[ii, jj]) || Double.IsInfinity(eval.Hessian[ii, jj]))
+                        throw new EvaluationException("Non-finite Hessian returned.", eval);
+                }
+            }
+        }
+    }
+}

src/Numerics/Optimization/ObjectiveChecker.cs

@@ -0,0 +1,164 @@
+// <copyright file="ObjectiveChecker.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class CheckedEvaluation : IEvaluation
+    {
+        readonly ObjectiveChecker Checker;
+        public IEvaluation InnerEvaluation { get; private set; }
+        bool ValueChecked;
+        bool GradientChecked;
+        bool HessianChecked;
+
+        public CheckedEvaluation(ObjectiveChecker checker, IEvaluation evaluation)
+        {
+            Checker = checker;
+            InnerEvaluation = evaluation;
+        }
+
+        public Vector<double> Point
+        {
+            get { return InnerEvaluation.Point; }
+        }
+
+        public EvaluationStatus Status
+        {
+            get { return InnerEvaluation.Status; }
+        }
+
+        public double Value
+        {
+            get
+            {
+                if (!ValueChecked)
+                {
+                    double tmp;
+                    try
+                    {
+                        tmp = InnerEvaluation.Value;
+                    }
+                    catch (Exception e)
+                    {
+                        throw new EvaluationException("Objective function evaluation failed.", InnerEvaluation, e);
+                    }
+                    Checker.ValueChecker(InnerEvaluation);
+					ValueChecked = true;
+                }
+                return InnerEvaluation.Value;
+            }
+        }
+
+        public Vector<double> Gradient
+        {
+            get
+            {
+                if (!GradientChecked)
+                {
+                    Vector<double> tmp;
+                    try
+                    {
+                        tmp = InnerEvaluation.Gradient;
+                    }
+                    catch (Exception e)
+                    {
+                        throw new EvaluationException("Objective gradient evaluation failed.", InnerEvaluation, e);
+                    }
+                    Checker.GradientChecker(InnerEvaluation);
+					GradientChecked = true;
+                }
+                return InnerEvaluation.Gradient;
+            }
+        }
+
+        public Matrix<double> Hessian
+        {
+            get
+            {
+                if (!HessianChecked)
+                {
+                    Matrix<double> tmp;
+                    try
+                    {
+                        tmp = InnerEvaluation.Hessian;
+                    }
+                    catch (Exception e)
+                    {
+                        throw new EvaluationException("Objective hessian evaluation failed.", InnerEvaluation, e);
+                    }
+                    Checker.HessianChecker(InnerEvaluation);
+					HessianChecked = true;
+                }
+                return InnerEvaluation.Hessian;
+            }
+        }
+    }
+
+    public class ObjectiveChecker : IObjectiveFunction
+    {
+        public IObjectiveFunction InnerObjective { get; private set; }
+        public Action<IEvaluation> ValueChecker { get; private set; }
+        public Action<IEvaluation> GradientChecker { get; private set; }
+        public Action<IEvaluation> HessianChecker { get; private set; }
+
+        public ObjectiveChecker(IObjectiveFunction objective, Action<IEvaluation> valueChecker, Action<IEvaluation> gradientChecker, Action<IEvaluation> hessianChecker)
+        {
+            InnerObjective = objective;
+            ValueChecker = valueChecker;
+            GradientChecker = gradientChecker;
+            HessianChecker = hessianChecker;
+        }
+
+        public bool GradientSupported
+        {
+            get { return InnerObjective.GradientSupported; }
+        }
+
+        public bool HessianSupported
+        {
+            get { return InnerObjective.HessianSupported; }
+        }
+
+        public IEvaluation Evaluate(Vector<double> point)
+        {
+            try
+            {
+                return new CheckedEvaluation(this, InnerObjective.Evaluate(point));
+            }
+            catch (Exception e)
+            {
+                throw new EvaluationException("Objective evaluation failed.", new NullEvaluation(point), e);
+            }
+        }
+    }
+}

src/Numerics/Optimization/ObjectiveChecker1D.cs

@@ -0,0 +1,163 @@
+// <copyright file="ObjectiveChecker1D.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class CheckedEvaluation1D : IEvaluation1D
+    {
+        readonly ObjectiveChecker1D Checker;
+        readonly IEvaluation1D InnerEvaluation;
+        bool ValueChecked;
+        bool DerivativeChecked;
+        bool SecondDerivativeChecked;
+
+        public CheckedEvaluation1D(ObjectiveChecker1D checker, IEvaluation1D evaluation)
+        {
+            Checker = checker;
+            InnerEvaluation = evaluation;
+        }
+
+        public double Point
+        {
+            get { return InnerEvaluation.Point; }
+        }
+
+        public EvaluationStatus Status
+        {
+            get { return InnerEvaluation.Status; }
+        }
+
+        public double Value
+        {
+            get
+            {
+                if (!ValueChecked)
+                {
+                    double tmp;
+                    try
+                    {
+                        tmp = InnerEvaluation.Value;
+                    }
+                    catch (Exception e)
+                    {
+                        throw new EvaluationException("Objective function evaluation failed.", InnerEvaluation, e);
+                    }
+                    Checker.ValueChecker(InnerEvaluation);
+					ValueChecked = true;
+                }
+                return InnerEvaluation.Value;
+            }
+        }
+
+        public double Derivative
+        {
+            get
+            {
+                if (!DerivativeChecked)
+                {
+                    double tmp;
+                    try
+                    {
+                        tmp = InnerEvaluation.Derivative;
+                    }
+                    catch (Exception e)
+                    {
+                        throw new EvaluationException("Objective derivative evaluation failed.", InnerEvaluation, e);
+                    }
+                    Checker.DerivativeChecker(InnerEvaluation);
+					DerivativeChecked = true;
+                }
+                return InnerEvaluation.Derivative;
+            }
+        }
+
+        public double SecondDerivative
+        {
+            get
+            {
+                if (!SecondDerivativeChecked)
+                {
+                    double tmp;
+                    try
+                    {
+                        tmp = InnerEvaluation.SecondDerivative;
+                    }
+                    catch (Exception e)
+                    {
+                        throw new EvaluationException("Objective second derivative evaluation failed.", InnerEvaluation, e);
+                    }
+                    Checker.SecondDerivativeChecker(InnerEvaluation);
+					SecondDerivativeChecked = true;
+                }
+                return InnerEvaluation.SecondDerivative;
+            }
+        }
+    }
+
+    public class ObjectiveChecker1D : IObjectiveFunction1D
+    {
+        public IObjectiveFunction1D InnerObjective { get; private set; }
+        public Action<IEvaluation1D> ValueChecker { get; private set; }
+        public Action<IEvaluation1D> DerivativeChecker { get; private set; }
+        public Action<IEvaluation1D> SecondDerivativeChecker { get; private set; }
+
+        public ObjectiveChecker1D(IObjectiveFunction1D objective, Action<IEvaluation1D> valueChecker, Action<IEvaluation1D> gradientChecker, Action<IEvaluation1D> hessianChecker)
+        {
+            InnerObjective = objective;
+            ValueChecker = valueChecker;
+            DerivativeChecker = gradientChecker;
+            SecondDerivativeChecker = hessianChecker;
+        }
+
+        public bool DerivativeSupported
+        {
+            get { return InnerObjective.DerivativeSupported; }
+        }
+
+        public bool SecondDerivativeSupported
+        {
+            get { return InnerObjective.SecondDerivativeSupported; }
+        }
+
+        public IEvaluation1D Evaluate(double point)
+        {
+            try
+            {
+                return new CheckedEvaluation1D(this, InnerObjective.Evaluate(point));
+            }
+            catch (Exception e)
+            {
+                throw new EvaluationException("Objective evaluation failed.", new NullEvaluation(point), e);
+            }
+        }
+    }
+}

src/Numerics/Optimization/ObjectiveFunction.cs

@@ -0,0 +1,265 @@
+// <copyright file="ObjectiveFunction.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+using MathNet.Numerics.LinearAlgebra.Double;
+
+namespace MathNet.Numerics.Optimization
+{
+    [Flags]
+    public enum EvaluationStatus
+    {
+        None = 0,
+        Value = 1,
+        Gradient = 2,
+        Hessian = 4
+    }
+
+    public interface IEvaluation
+    {
+        Vector<double> Point { get; }
+        EvaluationStatus Status { get; }
+        double Value { get; }
+        Vector<double> Gradient { get; }
+        Matrix<double> Hessian { get; }
+    }
+
+    public interface IObjectiveFunction
+    {
+        bool GradientSupported { get; }
+        bool HessianSupported { get; }
+        IEvaluation Evaluate(Vector<double> point);
+    }
+
+    public abstract class BaseEvaluation : IEvaluation
+    {
+        protected EvaluationStatus _status;
+        protected double? _value;
+        protected Vector<double> _gradient;
+        protected Matrix<double> _hessian;
+        protected Vector<double> _point;
+
+        public Vector<double> Point
+        {
+            get { return _point; }
+        }
+
+        protected BaseEvaluation()
+        {
+            _status = EvaluationStatus.None;
+        }
+
+        public EvaluationStatus Status
+        {
+            get { return _status; }
+        }
+
+        public double Value
+        {
+            get
+            {
+                if (!_value.HasValue)
+                {
+                    SetValue();
+                    _status |= EvaluationStatus.Value;
+                }
+                return _value.Value;
+            }
+        }
+
+        public Vector<double> Gradient
+        {
+            get
+            {
+                if (_gradient == null)
+                {
+                    SetGradient();
+                    _status |= EvaluationStatus.Gradient;
+                }
+                return _gradient;
+            }
+        }
+
+        public Matrix<double> Hessian
+        {
+            get
+            {
+                if (_hessian == null)
+                {
+                    SetHessian();
+                    _status |= EvaluationStatus.Hessian;
+                }
+                return _hessian;
+            }
+        }
+
+        protected abstract void SetValue();
+        protected abstract void SetGradient();
+        protected abstract void SetHessian();
+    }
+
+
+    public class NullEvaluation : BaseEvaluation
+    {
+
+        public NullEvaluation(Vector<double> point)
+        {
+            _point = point;
+            _status = EvaluationStatus.None;
+        }
+
+        public NullEvaluation(double point)
+        {
+            _point = DenseVector.Create(1, point);
+            _status = EvaluationStatus.None;
+        }
+
+        protected override void SetValue()
+        {
+            throw new NotImplementedException();
+        }
+
+        protected override void SetGradient()
+        {
+            throw new NotImplementedException();
+        }
+
+        protected override void SetHessian()
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public class OneDEvaluationExpander : IEvaluation
+    {
+        public IEvaluation1D InnerEval { get; private set; }
+
+        public OneDEvaluationExpander(IEvaluation1D eval)
+        {
+            InnerEval = eval;
+        }
+
+
+        public Vector<double> Point
+        {
+            get { return DenseVector.Create(1, InnerEval.Point); }
+        }
+
+        public EvaluationStatus Status
+        {
+            get { return InnerEval.Status; }
+        }
+
+        public double Value
+        {
+            get { return InnerEval.Value; }
+        }
+
+        public Vector<double> Gradient
+        {
+            get { return DenseVector.Create(1, InnerEval.Derivative); }
+        }
+
+        public Matrix<double> Hessian
+        {
+            get { return DenseMatrix.Create(1, 1, InnerEval.SecondDerivative); }
+        }
+    }
+
+    public class CachedEvaluation : BaseEvaluation
+    {
+        readonly SimpleObjectiveFunction _objectiveObject;
+
+        public CachedEvaluation(SimpleObjectiveFunction f, Vector<double> point)
+        {
+            _objectiveObject = f;
+            _point = point;
+        }
+
+        protected override void SetValue()
+        {
+            _value = _objectiveObject.Objective(_point);
+        }
+
+        protected override void SetGradient()
+        {
+            _gradient = _objectiveObject.Gradient(_point);
+        }
+
+        protected override void SetHessian()
+        {
+            _hessian = _objectiveObject.Hessian(_point);
+        }
+    }
+
+    public class SimpleObjectiveFunction : IObjectiveFunction
+    {
+        public Func<Vector<double>, double> Objective { get; private set; }
+        public Func<Vector<double>, Vector<double>> Gradient { get; private set; }
+        public Func<Vector<double>, Matrix<double>> Hessian { get; private set; }
+
+        public SimpleObjectiveFunction(Func<Vector<double>, double> objective)
+        {
+            Objective = objective;
+            Gradient = null;
+            Hessian = null;
+        }
+
+        public SimpleObjectiveFunction(Func<Vector<double>, double> objective, Func<Vector<double>, Vector<double>> gradient)
+        {
+            Objective = objective;
+            Gradient = gradient;
+            Hessian = null;
+        }
+
+        public SimpleObjectiveFunction(Func<Vector<double>, double> objective, Func<Vector<double>, Vector<double>> gradient, Func<Vector<double>, Matrix<double>> hessian)
+        {
+            Objective = objective;
+            Gradient = gradient;
+            Hessian = hessian;
+        }
+
+        public bool GradientSupported
+        {
+            get { return Gradient != null; }
+        }
+
+        public bool HessianSupported
+        {
+            get { return Hessian != null; }
+        }
+
+        public IEvaluation Evaluate(Vector<double> point)
+        {
+            return new CachedEvaluation(this, point);
+        }
+    }
+}

src/Numerics/Optimization/ObjectiveFunction1D.cs

@@ -0,0 +1,186 @@
+// <copyright file="ObjectiveFunction1D.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Optimization
+{
+    public interface IEvaluation1D
+    {
+        double Point { get; }
+        EvaluationStatus Status { get; }
+        double Value { get; }
+        double Derivative { get; }
+        double SecondDerivative { get; }
+    }
+
+    public interface IObjectiveFunction1D
+    {
+        bool DerivativeSupported { get; }
+        bool SecondDerivativeSupported { get; }
+        IEvaluation1D Evaluate(double point);
+    }
+
+    public abstract class BaseEvaluation1D : IEvaluation1D
+    {
+        protected double _point;
+        protected EvaluationStatus _status;
+        protected double? _value;
+        protected double? _derivative;
+        protected double? _secondDerivative;
+
+        public double Point
+        {
+            get { return _point; }
+        }
+
+        protected BaseEvaluation1D()
+        {
+            _status = EvaluationStatus.None;
+        }
+
+        public EvaluationStatus Status
+        {
+            get { return _status; }
+        }
+
+        public double Value
+        {
+            get
+            {
+                if (!_value.HasValue)
+                {
+                    SetValue();
+                    _status |= EvaluationStatus.Value;
+                }
+                return _value.Value;
+            }
+        }
+
+        public double Derivative
+        {
+            get
+            {
+                if (_derivative == null)
+                {
+                    SetDerivative();
+                    _status |= EvaluationStatus.Gradient;
+                }
+                return _derivative.Value;
+            }
+        }
+
+        public double SecondDerivative
+        {
+            get
+            {
+                if (_secondDerivative == null)
+                {
+                    SetSecondDerivative();
+                    _status |= EvaluationStatus.Hessian;
+                }
+                return _secondDerivative.Value;
+            }
+        }
+
+        protected abstract void SetValue();
+        protected abstract void SetDerivative();
+        protected abstract void SetSecondDerivative();
+    }
+
+    public class CachedEvaluation1D : BaseEvaluation1D
+    {
+        readonly SimpleObjectiveFunction1D _objectiveObject;
+
+        public CachedEvaluation1D(SimpleObjectiveFunction1D f, double point)
+        {
+            _objectiveObject = f;
+            _point = point;
+        }
+
+        protected override void SetValue()
+        {
+            _value = _objectiveObject.Objective(_point);
+        }
+
+        protected override void SetDerivative()
+        {
+            _derivative = _objectiveObject.Derivative(_point);
+        }
+
+        protected override void SetSecondDerivative()
+        {
+            _secondDerivative = _objectiveObject.SecondDerivative(_point);
+        }
+    }
+
+    public class SimpleObjectiveFunction1D : IObjectiveFunction1D
+    {
+        public Func<double, double> Objective { get; private set; }
+        public Func<double, double> Derivative { get; private set; }
+        public Func<double, double> SecondDerivative { get; private set; }
+
+        public SimpleObjectiveFunction1D(Func<double, double> objective)
+        {
+            Objective = objective;
+            Derivative = null;
+            SecondDerivative = null;
+        }
+
+        public SimpleObjectiveFunction1D(Func<double, double> objective, Func<double, double> derivative)
+        {
+            Objective = objective;
+            Derivative = derivative;
+            SecondDerivative = null;
+        }
+
+        public SimpleObjectiveFunction1D(Func<double, double> objective, Func<double, double> derivative, Func<double, double> secondDerivative)
+        {
+            Objective = objective;
+            Derivative = derivative;
+            SecondDerivative = secondDerivative;
+        }
+
+        public bool DerivativeSupported
+        {
+            get { return Derivative != null; }
+        }
+
+        public bool SecondDerivativeSupported
+        {
+            get { return SecondDerivative != null; }
+        }
+
+        public IEvaluation1D Evaluate(double point)
+        {
+            return new CachedEvaluation1D(this, point);
+        }
+    }
+}

src/Numerics/Optimization/WeakWolfeLineSearch.cs

@@ -0,0 +1,143 @@
+// <copyright file="ObjectiveFunction1D.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.Optimization
+{
+    public class WeakWolfeLineSearch
+    {
+        public double C1 { get; set; }
+        public double C2 { get; set; }
+        public double ParameterTolerance { get; set; }
+        public int MaximumIterations { get; set; }
+
+        public WeakWolfeLineSearch(double c1, double c2, double parameterTolerance, int maxIterations = 10)
+        {
+            C1 = c1;
+            C2 = c2;
+            ParameterTolerance = parameterTolerance;
+            MaximumIterations = maxIterations;
+        }
+
+        // Implemented following http://www.math.washington.edu/~burke/crs/408/lectures/L9-weak-Wolfe.pdf
+        public LineSearchOutput FindConformingStep(IObjectiveFunction objective, IEvaluation startingPoint, Vector<double> searchDirection, double initialStep)
+        {
+
+            if (!(objective is ObjectiveChecker))
+                objective = new ObjectiveChecker(objective, ValidateValue, ValidateGradient, null);
+
+            double lowerBound = 0.0;
+            double upperBound = Double.PositiveInfinity;
+            double step = initialStep;
+
+            double initialValue = startingPoint.Value;
+            Vector<double> initialGradient = startingPoint.Gradient;
+
+            double initialDd = searchDirection*initialGradient;
+
+            int ii;
+            IEvaluation candidateEval = null;
+            var reasonForExit = ExitCondition.None;
+            for (ii = 0; ii < MaximumIterations; ++ii)
+            {
+                candidateEval = objective.Evaluate(startingPoint.Point + searchDirection*step);
+
+                double stepDd = searchDirection*candidateEval.Gradient;
+
+                if (candidateEval.Value > initialValue + C1*step*initialDd)
+                {
+                    upperBound = step;
+                    step = 0.5*(lowerBound + upperBound);
+                }
+                else if (stepDd < C2*initialDd)
+                {
+                    lowerBound = step;
+                    step = Double.IsPositiveInfinity(upperBound) ? 2*lowerBound : 0.5*(lowerBound + upperBound);
+                }
+                else
+                {
+                    reasonForExit = ExitCondition.WeakWolfeCriteria;
+                    break;
+                }
+
+                if (!Double.IsInfinity(upperBound))
+                {
+                    double maxRelChange = 0.0;
+                    for (int jj = 0; jj < candidateEval.Point.Count; ++jj)
+                    {
+                        double tmp = Math.Abs(searchDirection[jj]*(upperBound - lowerBound))/Math.Max(Math.Abs(candidateEval.Point[jj]), 1.0);
+                        maxRelChange = Math.Max(maxRelChange, tmp);
+                    }
+                    if (maxRelChange < ParameterTolerance)
+                    {
+                        reasonForExit = ExitCondition.LackOfProgress;
+                        break;
+                    }
+                }
+            }
+
+            if (ii == MaximumIterations && Double.IsPositiveInfinity(upperBound))
+                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function appears to be unbounded in search direction.", MaximumIterations));
+            if (ii == MaximumIterations)
+                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
+            return new LineSearchOutput(candidateEval, ii, step, reasonForExit);
+        }
+
+        bool Conforms(IEvaluation startingPoint, Vector<double> searchDirection, double step, IEvaluation endingPoint)
+        {
+            bool sufficientDecrease = endingPoint.Value <= startingPoint.Value + C1*step*(startingPoint.Gradient*searchDirection);
+            bool notTooSteep = endingPoint.Gradient*searchDirection >= C2*startingPoint.Gradient*searchDirection;
+
+            return step > 0 && sufficientDecrease && notTooSteep;
+        }
+
+        void ValidateValue(IEvaluation eval)
+        {
+            if (!IsFinite(eval.Value))
+                throw new EvaluationException(String.Format("Non-finite value returned by objective function: {0}", eval.Value), eval);
+        }
+
+        void ValidateGradient(IEvaluation eval)
+        {
+            foreach (double x in eval.Gradient)
+                if (!IsFinite(x))
+                {
+                    throw new EvaluationException(String.Format("Non-finite value returned by gradient: {0}", x), eval);
+                }
+        }
+
+        bool IsFinite(double x)
+        {
+            return !(Double.IsNaN(x) || Double.IsInfinity(x));
+        }
+    }
+}

src/UnitTests/OptimizationTests/RosenbrockFunction.cs

@@ -0,0 +1,81 @@
+// <copyright file="RosenbrockFunction" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    public static class RosenbrockFunction
+    {
+        public static double Value(Vector<double> input)
+        {
+            return Math.Pow((1 - input[0]), 2) + 100*Math.Pow((input[1] - input[0]*input[0]), 2);
+        }
+
+        public static Vector<double> Gradient(Vector<double> input)
+        {
+            Vector<double> output = new LinearAlgebra.Double.DenseVector(2);
+            output[0] = -2*(1 - input[0]) + 200*(input[1] - input[0]*input[0])*(-2*input[0]);
+            output[1] = 2*100*(input[1] - input[0]*input[0]);
+            return output;
+        }
+
+        public static Matrix<double> Hessian(Vector<double> input)
+        {
+
+            Matrix<double> output = new LinearAlgebra.Double.DenseMatrix(2, 2);
+            output[0, 0] = 2 - 400*input[1] + 1200*input[0]*input[0];
+            output[1, 1] = 200;
+            output[0, 1] = -400*input[0];
+            output[1, 0] = output[0, 1];
+            return output;
+        }
+    }
+
+	public static class BigRosenbrockFunction
+	{
+		public static double Value(Vector<double> input)
+		{
+			return 1000.0 + 100.0*RosenbrockFunction.Value(input / 100.0);
+		}
+
+		public static Vector<double> Gradient(Vector<double> input)
+		{
+			return 100.0* RosenbrockFunction.Gradient(input / 100.0);
+		}
+
+		public static Matrix<double> Hessian(Vector<double> input)
+		{
+			return 100.0*RosenbrockFunction.Hessian(input / 100.0);
+		}
+
+	}
+}

src/UnitTests/OptimizationTests/TestBfgsMinimizer.cs

@@ -0,0 +1,108 @@
+// <copyright file="TestBfgsMinimizer" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using NUnit.Framework;
+using MathNet.Numerics.Optimization;
+using System.Threading;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    [TestFixture]
+    public class TestBfgsMinimizer
+    {
+        [Test]
+        public void FindMinimum_Rosenbrock_Easy()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+            var solver = new BfgsMinimizer(1e-5, 1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { 1.2, 1.2 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+			Thread.Sleep(10*1000);
+        }
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Hard()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+            var solver = new BfgsMinimizer(1e-5, 1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -1.2, 1.0 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Overton()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+            var solver = new BfgsMinimizer(1e-5, 1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -0.9, -0.5 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+		[Test]
+		public void FindMinimum_BigRosenbrock_Easy()
+		{
+			var obj = new SimpleObjectiveFunction(BigRosenbrockFunction.Value, BigRosenbrockFunction.Gradient);
+			var solver = new BfgsMinimizer(1e-10, 1e-5, 1000);
+			var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { 1.2*100.0, 1.2*100.0 }));
+			
+			Assert.That(Math.Abs(result.MinimizingPoint[0] - 100.0), Is.LessThan(1e-3));
+			Assert.That(Math.Abs(result.MinimizingPoint[1] - 100.0), Is.LessThan(1e-3));
+		}
+
+		[Test]
+		public void FindMinimum_BigRosenbrock_Hard()
+		{
+			var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+			var solver = new BfgsMinimizer(1e-5, 1e-5, 1000);
+			var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -1.2*100.0, 1.0*100.0 }));
+			
+			Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+			Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+		}
+		
+		[Test]
+		public void FindMinimum_BigRosenbrock_Overton()
+		{
+			var obj = new SimpleObjectiveFunction(BigRosenbrockFunction.Value, BigRosenbrockFunction.Gradient);
+			var solver = new BfgsMinimizer(1e-5, 1e-5, 1000);
+			var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -0.9*100.0, -0.5*100.0 }));
+			
+			Assert.That(Math.Abs(result.MinimizingPoint[0] - 100.0), Is.LessThan(1e-3));
+			Assert.That(Math.Abs(result.MinimizingPoint[1] - 100.0), Is.LessThan(1e-3));
+		}
+    }
+}

src/UnitTests/OptimizationTests/TestBisectionRootFinder.cs

@@ -0,0 +1,82 @@
+// <copyright file="TestBisectionRootFinder" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using NUnit.Framework;
+using MathNet.Numerics.Optimization;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    [TestFixture]
+    internal class TestBisectionRootFinder
+    {
+        [Test]
+        public void FindRoot_Works()
+        {
+            var algorithm = new BisectionRootFinder(0.001, 0.001);
+            var f1 = new Func<double, double>(x => (x - 3)*(x - 4));
+            var r1 = algorithm.FindRoot(f1, 2.1, 3.9);
+            Assert.That(Math.Abs(f1(r1)), Is.LessThan(0.001));
+            Assert.That(Math.Abs(r1 - 3.0), Is.LessThan(0.001));
+
+            var f2 = new Func<double, double>(x => (x - 3)*(x - 4));
+            var r2 = algorithm.FindRoot(f1, 2.1, 3.4);
+            Assert.That(Math.Abs(f2(r2)), Is.LessThan(0.001));
+            Assert.That(Math.Abs(r2 - 3.0), Is.LessThan(0.001));
+        }
+
+		[Test]
+		public void FindRoot_ConstantZeroRegion_HitsFirstZeroOnExpansion()
+		{
+			var algorithm = new BisectionRootFinder(1e-3,1e-3,lowerExpansionFactor:2,upperExpansionFactor:2);
+
+			var r1 = algorithm.FindRoot(function_goes_flat_at_zero, -1,0);
+
+			Assert.That(Math.Abs(r1 - 1), Is.LessThan(1e-3));
+
+		}
+
+		[Test]
+		public void FindRoot_ConstantZeroRegion_HitsPastFirstZeroOnExpansion()
+		{
+			var algorithm = new BisectionRootFinder(1e-3,1e-3,lowerExpansionFactor:2,upperExpansionFactor:2);
+			
+			var r1 = algorithm.FindRoot(function_goes_flat_at_zero, -2.25,0);
+			
+			Assert.That(Math.Abs(r1 - 1), Is.LessThan(1e-3));
+			
+		}
+
+		private static double function_goes_flat_at_zero(double x)
+		{
+			return x < 1 ? -3*(x - 1) : 0.0;
+		}
+    }
+}

src/UnitTests/OptimizationTests/TestConjugateGradientMinimizer.cs

@@ -0,0 +1,62 @@
+// <copyright file="TestConjugateGradientMinimizer" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using NUnit.Framework;
+using MathNet.Numerics.Optimization;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    [TestFixture]
+    public class TestConjugateGradientMinimizer
+    {
+        [Test]
+        public void FindMinimum_Rosenbrock_Easy()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+            var solver = new ConjugateGradientMinimizer(1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { 1.2, 1.2 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Hard()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient);
+            var solver = new ConjugateGradientMinimizer(1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -1.2, 1.0 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+    }
+}

src/UnitTests/OptimizationTests/TestNewtonMinimizer.cs

@@ -0,0 +1,106 @@
+// <copyright file="TestNewtonMinimizer" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using NUnit.Framework;
+using MathNet.Numerics.Optimization;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    [TestFixture]
+    public class TestNewtonMinimizer
+    {
+        [Test]
+        public void FindMinimum_Rosenbrock_Easy()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient, RosenbrockFunction.Hessian);
+            var solver = new NewtonMinimizer(1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { 1.2, 1.2 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Hard()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient, RosenbrockFunction.Hessian);
+            var solver = new NewtonMinimizer(1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -1.2, 1.0 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Rosenbrock_Overton()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient, RosenbrockFunction.Hessian);
+            var solver = new NewtonMinimizer(1e-5, 1000);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -0.9, -0.5 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Linesearch_Rosenbrock_Easy()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient, RosenbrockFunction.Hessian);
+            var solver = new NewtonMinimizer(1e-5, 1000, true);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { 1.2, 1.2 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Linesearch_Rosenbrock_Hard()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient, RosenbrockFunction.Hessian);
+            var solver = new NewtonMinimizer(1e-5, 1000, true);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -1.2, 1.0 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+
+        [Test]
+        public void FindMinimum_Linesearch_Rosenbrock_Overton()
+        {
+            var obj = new SimpleObjectiveFunction(RosenbrockFunction.Value, RosenbrockFunction.Gradient, RosenbrockFunction.Hessian);
+            var solver = new NewtonMinimizer(1e-5, 1000, true);
+            var result = solver.FindMinimum(obj, new LinearAlgebra.Double.DenseVector(new[] { -0.9, -0.5 }));
+
+            Assert.That(Math.Abs(result.MinimizingPoint[0] - 1.0), Is.LessThan(1e-3));
+            Assert.That(Math.Abs(result.MinimizingPoint[1] - 1.0), Is.LessThan(1e-3));
+        }
+    }
+}

src/UnitTests/OptimizationTests/TestRosenbrockFunction.cs

@@ -0,0 +1,86 @@
+// <copyright file="TestRosenbrockFunction" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2013 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using NUnit.Framework;
+
+namespace MathNet.Numerics.UnitTests.OptimizationTests
+{
+    [TestFixture]
+    internal class TestRosenbrockFunction
+    {
+        [Test]
+        public void TestGradient()
+        {
+            var input = new LinearAlgebra.Double.DenseVector(new[] { -0.9, -0.5 });
+
+            var v1 = RosenbrockFunction.Value(input);
+            var g = RosenbrockFunction.Gradient(input);
+
+            const double eps = 1e-5;
+            var eps0 = (new LinearAlgebra.Double.DenseVector(new[] { 1.0, 0.0 }))*eps;
+            var eps1 = (new LinearAlgebra.Double.DenseVector(new[] { 0.0, 1.0 }))*eps;
+
+            var g0 = (RosenbrockFunction.Value(input + eps0) - RosenbrockFunction.Value(input - eps0))/(2*eps);
+            var g1 = (RosenbrockFunction.Value(input + eps1) - RosenbrockFunction.Value(input - eps1))/(2*eps);
+
+            Assert.That(Math.Abs(g0 - g[0]) < 1e-3);
+            Assert.That(Math.Abs(g1 - g[1]) < 1e-3);
+        }
+
+        [Test]
+        public void TestHessian()
+        {
+            var input = new LinearAlgebra.Double.DenseVector(new[] { -0.9, -0.5 });
+
+            var v1 = RosenbrockFunction.Value(input);
+            var h = RosenbrockFunction.Hessian(input);
+
+            const double eps = 1e-5;
+
+            var eps0 = (new LinearAlgebra.Double.DenseVector(new[] { 1.0, 0.0 }))*eps;
+            var eps1 = (new LinearAlgebra.Double.DenseVector(new[] { 0.0, 1.0 }))*eps;
+
+            var epsuu = (new LinearAlgebra.Double.DenseVector(new[] { 1.0, 1.0 }))*eps;
+            var epsud = (new LinearAlgebra.Double.DenseVector(new[] { 1.0, -1.0 }))*eps;
+
+
+            var h00 = (RosenbrockFunction.Value(input + eps0) - 2*RosenbrockFunction.Value(input) + RosenbrockFunction.Value(input - eps0))/(eps*eps);
+            var h11 = (RosenbrockFunction.Value(input + eps1) - 2*RosenbrockFunction.Value(input) + RosenbrockFunction.Value(input - eps1))/(eps*eps);
+            var h01 = (RosenbrockFunction.Value(input + epsuu) - RosenbrockFunction.Value(input + epsud) - RosenbrockFunction.Value(input - epsud) + RosenbrockFunction.Value(input - epsuu))/(4*eps*eps);
+
+
+            Assert.That(Math.Abs(h00 - h[0, 0]) < 1e-3);
+            Assert.That(Math.Abs(h11 - h[1, 1]) < 1e-3);
+            Assert.That(Math.Abs(h01 - h[0, 1]) < 1e-3);
+            Assert.That(Math.Abs(h01 - h[1, 0]) < 1e-3);
+        }
+    }
+}

src/UnitTests/UnitTests.csproj

@@ -360,6 +360,12 @@
     <Compile Include="EuclidTests\IntegerTheoryTest.cs" />
     <Compile Include="Random\SystemRandomSourceTests.cs" />
     <Compile Include="RootFindingTests\BisectionTest.cs" />
+    <Compile Include="OptimizationTests\RosenbrockFunction.cs" />
+    <Compile Include="OptimizationTests\TestBfgsMinimizer.cs" />
+    <Compile Include="OptimizationTests\TestBisectionRootFinder.cs" />
+    <Compile Include="OptimizationTests\TestConjugateGradientMinimizer.cs" />
+    <Compile Include="OptimizationTests\TestNewtonMinimizer.cs" />
+    <Compile Include="OptimizationTests\TestRosenbrockFunction.cs" />
     <Compile Include="PermutationTest.cs" />
     <Compile Include="PrecisionTest.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -682,5 +688,6 @@
   <ItemGroup>
     <Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
 </Project>