Optimization: Add the point which was being evaluated to EvaluationError

(updated by cdrnet)

src/Numerics/Optimization/BfgsMinimizer.cs

@@ -6,14 +6,17 @@ using MathNet.Numerics.LinearAlgebra;
 
 namespace MathNet.Numerics.Optimization
 {
+
     public class BfgsMinimizer
     {
         public double GradientTolerance { get; set; }
+		public double ParameterTolerance { get; set; }
         public int MaximumIterations { get; set; }
 
-        public BfgsMinimizer(double gradient_tolerance, int maximum_iterations)
+        public BfgsMinimizer(double gradient_tolerance, double parameter_tolerance, int maximum_iterations)
         {
             this.GradientTolerance = gradient_tolerance;
+			this.ParameterTolerance = parameter_tolerance;
             this.MaximumIterations = maximum_iterations;
         }
 
@@ -28,16 +31,17 @@ namespace MathNet.Numerics.Optimization
             IEvaluation initial_eval = objective.Evaluate(initial_guess);            
             
             // Check that we're not already done
-            if (this.ExitCriteriaSatisfied(initial_guess, initial_eval.Gradient))
-                return new MinimizationOutput(initial_eval, 0);
+			ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null);
+            if (current_exit_condition != ExitCondition.None)
+                return new MinimizationOutput(initial_eval, 0, current_exit_condition);
             
             // Set up line search algorithm
-            var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, 1000);
+            var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9,this.ParameterTolerance, max_iterations:1000);
 
             // Declare state variables
-            IEvaluation candidate_point;
+            IEvaluation candidate_point, previous_point;
             double step_size;
-            Vector<double> gradient, previous_gradient, step, search_direction;
+            Vector<double> gradient, step, search_direction;
             Matrix<double> inverse_pseudo_hessian;
 
             // First step
@@ -55,27 +59,27 @@ namespace MathNet.Numerics.Optimization
                 throw new InnerOptimizationException("Line search failed.", e);
             }
 
+			previous_point = initial_eval;
             candidate_point = result.FunctionInfoAtMinimum;
-            gradient = candidate_point.Gradient;
-            previous_gradient = initial_eval.Gradient;
+            gradient = candidate_point.Gradient;            
             step = candidate_point.Point - initial_guess;
             step_size = result.FinalStep;
 
             // Subsequent steps
-            int iterations = 1;
+            int iterations;
             int total_line_search_steps = result.Iterations;
             int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1;
             int steepest_descent_resets = 0;
-            while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations)
+			for (iterations = 1; iterations < this.MaximumIterations; ++iterations)
             {
-                var y = candidate_point.Gradient - previous_gradient;
+                var y = candidate_point.Gradient - previous_point.Gradient;
 
                 double sy = step * y;
                 inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ( (inverse_pseudo_hessian * y.ToColumnMatrix())*step.ToRowMatrix() + step.ToColumnMatrix()*(y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy);
 
-               search_direction = -inverse_pseudo_hessian * candidate_point.Gradient;
+                search_direction = -inverse_pseudo_hessian * candidate_point.Gradient;
 
-                if (search_direction * candidate_point.Gradient >= 0)
+                if (search_direction * candidate_point.Gradient >= -this.GradientTolerance*this.GradientTolerance)
                 {
                     search_direction = -candidate_point.Gradient;
                     inverse_pseudo_hessian = Matrix<double>.Build.DiagonalIdentity(initial_guess.Count);
@@ -96,21 +100,50 @@ namespace MathNet.Numerics.Optimization
 
                 step_size = result.FinalStep;
                 step = result.FunctionInfoAtMinimum.Point - candidate_point.Point;
-                previous_gradient = candidate_point.Gradient;
+				previous_point = candidate_point;
                 candidate_point = result.FunctionInfoAtMinimum;
 
-                iterations += 1;
+				current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point);
+				if (current_exit_condition != ExitCondition.None)
+					break;
             }
 
             if (iterations == this.MaximumIterations)
                 throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
 
-            return new MinimizationWithLineSearchOutput(candidate_point, iterations, total_line_search_steps, iterations_with_nontrivial_line_search);
+			return new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search);
         }
 
-        private bool ExitCriteriaSatisfied(Vector<double> candidate_point, Vector<double> gradient)
+        private ExitCondition ExitCriteriaSatisfied(IEvaluation candidate_point, IEvaluation last_point)
         {
-            return gradient.Norm(2.0) < this.GradientTolerance;
+			Vector<double> rel_grad = new MathNet.Numerics.LinearAlgebra.Double.DenseVector(candidate_point.Point.Count);
+			double relative_gradient = 0.0;
+			double normalizer = Math.Max(Math.Abs(candidate_point.Value),1.0);
+			for (int ii = 0; ii < rel_grad.Count; ++ii)
+			{
+				double tmp = candidate_point.Gradient[ii]*Math.Max(Math.Abs(candidate_point.Point[ii]), 1.0) / normalizer;
+				relative_gradient = Math.Max(relative_gradient, Math.Abs(tmp));
+			}
+			if (relative_gradient < this.GradientTolerance)
+			{
+				return ExitCondition.RelativeGradient;
+			}
+
+			if (last_point != null)
+			{
+				double most_progress = 0.0;
+				for (int ii = 0; ii < candidate_point.Point.Count; ++ii)
+				{
+					var tmp = Math.Abs(candidate_point.Point[ii] - last_point.Point[ii])/Math.Max(Math.Abs(last_point.Point[ii]),1.0);
+					most_progress = Math.Max(most_progress, tmp);
+				}
+				if ( most_progress < this.ParameterTolerance )
+				{
+					return ExitCondition.LackOfProgress;
+				}
+			}
+
+			return ExitCondition.None;
         }
 
         private void ValidateGradient(Vector<double> gradient, Vector<double> input)
@@ -118,14 +151,14 @@ namespace MathNet.Numerics.Optimization
             foreach (var x in gradient)
             {
                 if (Double.IsNaN(x) || Double.IsInfinity(x))
-                    throw new EvaluationException("Non-finite gradient returned.");
+                    throw new EvaluationException("Non-finite gradient returned.",input);
             }
         }
 
         private void ValidateObjective(double objective, Vector<double> input)
         {
             if (Double.IsNaN(objective) || Double.IsInfinity(objective))
-                throw new EvaluationException("Non-finite objective function returned.");
+                throw new EvaluationException("Non-finite objective function returned.", input);
         }
     }
 }

src/Numerics/Optimization/ConjugateGradientMinimizer.cs

@@ -31,10 +31,10 @@ namespace MathNet.Numerics.Optimization
             
             // Check that we're not already done
             if (this.ExitCriteriaSatisfied(initial_guess, gradient))
-                return new MinimizationOutput(initial_eval, 0);
+                return new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient);
             
             // Set up line search algorithm
-            var line_searcher = new WeakWolfeLineSearch(1e-4, 0.1,1000);
+            var line_searcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, max_iterations:1000);
 
             // Declare state variables
             IEvaluation candidate_point;
@@ -106,7 +106,7 @@ namespace MathNet.Numerics.Optimization
             if (iterations == this.MaximumIterations)
                 throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
 
-            return new MinimizationWithLineSearchOutput(candidate_point, iterations, total_line_search_steps, iterations_with_nontrivial_line_search);
+            return new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search);
         }
 
         private bool ExitCriteriaSatisfied(Vector<double> candidate_point, Vector<double> gradient)
@@ -119,14 +119,14 @@ namespace MathNet.Numerics.Optimization
             foreach (var x in gradient)
             {
                 if (Double.IsNaN(x) || Double.IsInfinity(x))
-                    throw new EvaluationException("Non-finite gradient returned.");
+                    throw new EvaluationException("Non-finite gradient returned.", input);
             }
         }
 
         private void ValidateObjective(double objective, Vector<double> input)
         {
             if (Double.IsNaN(objective) || Double.IsInfinity(objective))
-                throw new EvaluationException("Non-finite objective function returned.");
+                throw new EvaluationException("Non-finite objective function returned.", input);
         }
     }
 }

src/Numerics/Optimization/Exceptions.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using MathNet.Numerics.LinearAlgebra;
+using MathNet.Numerics.LinearAlgebra.Double;
 
 namespace MathNet.Numerics.Optimization
 {
@@ -22,11 +24,24 @@ namespace MathNet.Numerics.Optimization
 
     public class EvaluationException : OptimizationException
     {
-        public EvaluationException(string message)
-            : base(message) {}
-
-        public EvaluationException(string message, Exception inner_exception) 
-            : base(message, inner_exception) { }
+        public Vector<double> Point { get; private set; }
+        public EvaluationException(string message, Vector<double> point)
+            : base(message) 
+        {
+            this.Point = point;
+        }
+
+        public EvaluationException(string message, double point)
+            : base(message)
+        {
+            this.Point = new DenseVector(1, point);
+        }
+
+        public EvaluationException(string message, Exception inner_exception, Vector<double> point) 
+            : base(message, inner_exception) 
+        {
+            this.Point = point;
+        }
     }
 
     public class InnerOptimizationException : OptimizationException

src/Numerics/Optimization/ExitCondition.cs

@@ -0,0 +1,7 @@
+using System;
+
+namespace MathNet.Numerics
+{
+	public enum ExitCondition { None, RelativeGradient, LackOfProgress, AbsoluteGradient, WeakWolfeCriteria }
+}
+

src/Numerics/Optimization/GoldenSectionMinimizer.cs

@@ -66,7 +66,7 @@ namespace MathNet.Numerics.Optimization
         private void ValueChecker(double value, double point)
         {
             if (Double.IsNaN(value) || Double.IsInfinity(value))
-                throw new EvaluationException("Objective function returned non-finite value.");
+                throw new EvaluationException("Objective function returned non-finite value.", point);
         }
         private static double _golden_ratio = (1.0 + Math.Sqrt(5)) / 2.0;
     }

src/Numerics/Optimization/LineSearchOutput.cs

@@ -9,8 +9,8 @@ namespace MathNet.Numerics.Optimization
     {
         public double FinalStep { get; private set; }
 
-        public LineSearchOutput(IEvaluation function_info, int iterations, double final_step)
-            : base(function_info, iterations)
+        public LineSearchOutput(IEvaluation function_info, int iterations, double final_step, ExitCondition reason_for_exit)
+            : base(function_info, iterations, reason_for_exit)
         {
             this.FinalStep = final_step;
         }

src/Numerics/Optimization/LineSearchingMinimizerOutput.cs

@@ -10,8 +10,8 @@ namespace MathNet.Numerics.Optimization
         public int TotalLineSearchIterations { get; private set; }
         public int IterationsWithNonTrivialLineSearch { get; private set; }
 
-        public MinimizationWithLineSearchOutput(IEvaluation function_info, int iterations, int total_line_search_iterations, int iterations_with_non_trivial_line_search)
-            : base(function_info, iterations)
+        public MinimizationWithLineSearchOutput(IEvaluation function_info, int iterations, ExitCondition reason_for_exit, int total_line_search_iterations, int iterations_with_non_trivial_line_search)
+            : base(function_info, iterations, reason_for_exit)
         {
             this.TotalLineSearchIterations = total_line_search_iterations;
             this.IterationsWithNonTrivialLineSearch = iterations_with_non_trivial_line_search;

src/Numerics/Optimization/MinimizationOutput.cs

@@ -11,12 +11,14 @@ namespace MathNet.Numerics.Optimization
     {
         public Vector<double> MinimizingPoint { get { return FunctionInfoAtMinimum.Point; } }
         public IEvaluation FunctionInfoAtMinimum { get; private set; }
-        public int Iterations { get; private set; }
+        public int Iterations { get; private set; }		
+		public ExitCondition ReasonForExit { get; private set; }
 
-        public MinimizationOutput(IEvaluation function_info, int iterations)
+        public MinimizationOutput(IEvaluation function_info, int iterations, ExitCondition reason_for_exit)
         {
             this.FunctionInfoAtMinimum = function_info;
             this.Iterations = iterations;
+			this.ReasonForExit = reason_for_exit;
         }
     }
 }

src/Numerics/Optimization/NewtonMinimizer.cs

@@ -35,10 +35,10 @@ namespace MathNet.Numerics.Optimization
             
             // Check that we're not already done
             if (this.ExitCriteriaSatisfied(initial_guess, initial_eval.Gradient))
-                return new MinimizationOutput(initial_eval, 0);
+                return new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient);
             
             // Set up line search algorithm            
-            var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, 1000);
+            var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, max_iterations:1000);
 
             // Declare state variables
             IEvaluation candidate_point = initial_eval;            
@@ -90,7 +90,7 @@ namespace MathNet.Numerics.Optimization
             if (iterations == this.MaximumIterations)
                 throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
 
-            return new MinimizationWithLineSearchOutput(candidate_point, iterations, total_line_search_steps, iterations_with_nontrivial_line_search);
+            return new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search);
         }
 
         private bool ExitCriteriaSatisfied(Vector<double> candidate_point, Vector<double> gradient)
@@ -103,14 +103,14 @@ namespace MathNet.Numerics.Optimization
             foreach (var x in gradient)
             {
                 if (Double.IsNaN(x) || Double.IsInfinity(x))
-                    throw new EvaluationException("Non-finite gradient returned.");
+                    throw new EvaluationException("Non-finite gradient returned.", input);
             }
         }
 
         private void ValidateObjective(double objective, Vector<double> input)
         {
             if (Double.IsNaN(objective) || Double.IsInfinity(objective))
-                throw new EvaluationException("Non-finite objective function returned.");
+                throw new EvaluationException("Non-finite objective function returned.", input);
         }
 
         private void ValidateHessian(Matrix<double> hessian, Vector<double> input)
@@ -120,7 +120,7 @@ namespace MathNet.Numerics.Optimization
                 for (int jj = 0; jj < hessian.ColumnCount; ++jj)
                 {
                     if (Double.IsNaN(hessian[ii,jj]) || Double.IsInfinity(hessian[ii,jj]))
-                        throw new EvaluationException("Non-finite Hessian returned.");
+                        throw new EvaluationException("Non-finite Hessian returned.", input);
                 }
             }
         }

src/Numerics/Optimization/ObjectiveChecker.cs

@@ -39,7 +39,7 @@ namespace MathNet.Numerics.Optimization
                     }
                     catch (Exception e)
                     {
-                        throw new EvaluationException("Objective function evaluation failed.", e);
+                        throw new EvaluationException("Objective function evaluation failed.", e, this.Point);
                     }
                     this.Checker.ValueChecker(tmp,this.InnerEvaluation.Point);
                 }
@@ -61,7 +61,7 @@ namespace MathNet.Numerics.Optimization
                     }
                     catch (Exception e)
                     {
-                        throw new EvaluationException("Objective gradient evaluation failed.", e);
+                        throw new EvaluationException("Objective gradient evaluation failed.", e, this.Point);
                     }
                     this.Checker.GradientChecker(tmp, this.InnerEvaluation.Point);
                 }
@@ -83,7 +83,7 @@ namespace MathNet.Numerics.Optimization
                     }
                     catch (Exception e)
                     {
-                        throw new EvaluationException("Objective hessian evaluation failed.", e);
+                        throw new EvaluationException("Objective hessian evaluation failed.", e, this.Point);
                     }
                     this.Checker.HessianChecker(tmp, this.InnerEvaluation.Point);
                 }
@@ -125,7 +125,7 @@ namespace MathNet.Numerics.Optimization
             }
             catch (Exception e)
             {
-                throw new EvaluationException("Objective evaluation failed.", e);
+                throw new EvaluationException("Objective evaluation failed.", e, point);
             }
         }
     }

src/Numerics/Optimization/ObjectiveChecker1D.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using MathNet.Numerics.LinearAlgebra.Double;
 
 namespace MathNet.Numerics.Optimization
 {
@@ -38,7 +39,7 @@ namespace MathNet.Numerics.Optimization
                     }
                     catch (Exception e)
                     {
-                        throw new EvaluationException("Objective function evaluation failed.", e);
+                        throw new EvaluationException("Objective function evaluation failed.", e, new DenseVector(1, this.Point));
                     }
                     this.Checker.ValueChecker(tmp, this.InnerEvaluation.Point);
                 }
@@ -60,7 +61,7 @@ namespace MathNet.Numerics.Optimization
                     }
                     catch (Exception e)
                     {
-                        throw new EvaluationException("Objective derivative evaluation failed.", e);
+                        throw new EvaluationException("Objective derivative evaluation failed.", e, new DenseVector(1, this.Point));
                     }
                     this.Checker.DerivativeChecker(tmp, this.InnerEvaluation.Point);
                 }
@@ -82,7 +83,7 @@ namespace MathNet.Numerics.Optimization
                     }
                     catch (Exception e)
                     {
-                        throw new EvaluationException("Objective second derivative evaluation failed.", e);
+                        throw new EvaluationException("Objective second derivative evaluation failed.", e, new DenseVector(1,this.Point));
                     }
                     this.Checker.SecondDerivativeChecker(tmp, this.InnerEvaluation.Point);
                 }
@@ -124,7 +125,7 @@ namespace MathNet.Numerics.Optimization
             }
             catch (Exception e)
             {
-                throw new EvaluationException("Objective evaluation failed.", e);
+                throw new EvaluationException("Objective evaluation failed.", e, new DenseVector(1,point));
             }
         }
     }

src/Numerics/Optimization/ObjectiveFunction.cs

@@ -9,7 +9,7 @@ namespace MathNet.Numerics.Optimization
 {
     public interface IEvaluation
     {
-        Vector<double> Point { get; }
+        Vector<double> Point { get; }        
         double Value { get; }
         Vector<double> Gradient { get; }
         Matrix<double> Hessian { get; }
@@ -22,6 +22,48 @@ namespace MathNet.Numerics.Optimization
         IEvaluation Evaluate(Vector<double> point);
     }
 
+    public abstract class BaseEvaluation : IEvaluation
+    {
+        protected double? _value;
+        protected Vector<double> _gradient;
+        protected Matrix<double> _hessian;
+        protected Vector<double> _point;
+        
+        public Vector<double> Point { get { return _point; } }
+        public double Value 
+        { 
+            get 
+            {
+                if (!_value.HasValue)
+                    setValue();
+                return _value.Value; 
+            } 
+        }
+        public Vector<double> Gradient 
+        { 
+            get 
+            {
+                if (_gradient == null)
+                    setGradient();
+                return _gradient; 
+            } 
+        }
+        public Matrix<double> Hessian 
+        { 
+            get
+            {
+                if (_hessian == null)
+                    setHessian();
+                return _hessian; 
+            } 
+        }
+
+        protected abstract void setValue();
+        protected abstract void setGradient();
+        protected abstract void setHessian();
+    }
+
+
     public class CachedEvaluation : IEvaluation
     {
         private double? _value;

src/Numerics/Optimization/WeakWolfeLineSearch.cs

@@ -11,12 +11,14 @@ namespace MathNet.Numerics.Optimization
     {
         public double C1 { get; set; }
         public double C2 { get; set; }
+		public double ParameterTolerance { get; set; }
 		public int MaximumIterations { get; set; }
 
-        public WeakWolfeLineSearch(double c1, double c2, int max_iterations=10)
+        public WeakWolfeLineSearch(double c1, double c2, double parameter_tolerance, int max_iterations=10)
         {
             this.C1 = c1;
             this.C2 = c2;
+			this.ParameterTolerance = parameter_tolerance;
 			this.MaximumIterations = max_iterations;
         }
 
@@ -38,6 +40,7 @@ namespace MathNet.Numerics.Optimization
 
             int ii;
             IEvaluation candidate_eval = null;
+			ExitCondition reason_for_exit = ExitCondition.None;
             for (ii = 0; ii < this.MaximumIterations; ++ii)
             {
                 candidate_eval = objective.Evaluate(starting_point.Point + search_direction * step);
@@ -56,14 +59,32 @@ namespace MathNet.Numerics.Optimization
                 }
                 else 
                 {
+					reason_for_exit = ExitCondition.WeakWolfeCriteria;
                     break;
                 }
+
+				if (!Double.IsInfinity(upper_bound))
+				{
+					double max_rel_change = 0.0;
+					for (int jj = 0; jj < candidate_eval.Point.Count; ++jj)
+					{
+						double tmp = Math.Abs (search_direction[jj]*(upper_bound - lower_bound)) / Math.Max(Math.Abs(candidate_eval.Point[jj]),1.0);
+						max_rel_change = Math.Max(max_rel_change, tmp);
+					}
+					if (max_rel_change < this.ParameterTolerance)
+					{
+						reason_for_exit = ExitCondition.LackOfProgress;
+						break;
+					}
+				}
             }
                        
-            if (ii == this.MaximumIterations)
-                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function may be unbounded in search direction.",this.MaximumIterations));
+            if (ii == this.MaximumIterations && Double.IsPositiveInfinity(upper_bound))
+                throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function appears to be unbounded in search direction.",this.MaximumIterations));
+			else if (ii == this.MaximumIterations)
+				throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.",this.MaximumIterations));
             else
-                return new LineSearchOutput(candidate_eval, ii, step);
+                return new LineSearchOutput(candidate_eval, ii, step, reason_for_exit);
         }
 
         private bool Conforms(IEvaluation starting_point, Vector<double> search_direction, double step, IEvaluation ending_point)
@@ -78,7 +99,7 @@ namespace MathNet.Numerics.Optimization
         private void ValidateValue(double value, Vector<double> input)
         {
             if (!this.IsFinite(value))
-                throw new EvaluationException(String.Format("Non-finite value returned by objective function: {0}", value));
+                throw new EvaluationException(String.Format("Non-finite value returned by objective function: {0}", value),input);
         }
 
         private void ValidateGradient(Vector<double> gradient, Vector<double> input)
@@ -86,7 +107,7 @@ namespace MathNet.Numerics.Optimization
             foreach (double x in gradient)
                 if (!this.IsFinite(x))
                 {
-                    throw new EvaluationException(String.Format("Non-finite value returned by gradient: {0}", x));
+                    throw new EvaluationException(String.Format("Non-finite value returned by gradient: {0}", x),input);
                 }
         }