From e7ae7a6633a72d905c08cf0178b9d271339faecf Mon Sep 17 00:00:00 2001 From: Christoph Ruegg Date: Sat, 15 Jul 2017 21:22:34 +0200 Subject: [PATCH] Optimization: cleanup --- src/Numerics/FindMinimum.cs | 16 ++++----- src/Numerics/Optimization/BfgsBMinimizer.cs | 10 +++--- src/Numerics/Optimization/BfgsMinimizer.cs | 2 +- .../Optimization/BfgsMinimizerBase.cs | 2 +- .../ConjugateGradientMinimizer.cs | 34 ++++++++++++------- .../Optimization/GoldenSectionMinimizer.cs | 4 +-- .../Optimization/NelderMeadSimplex.cs | 6 ++-- src/Numerics/Optimization/NewtonMinimizer.cs | 4 +-- .../ConjugateGradientMinimizerTests.cs | 3 +- 9 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/Numerics/FindMinimum.cs b/src/Numerics/FindMinimum.cs index b32a59d2..28cfc154 100644 --- a/src/Numerics/FindMinimum.cs +++ b/src/Numerics/FindMinimum.cs @@ -43,8 +43,7 @@ namespace MathNet.Numerics public static double OfScalarFunctionConstrained(Func function, double lowerBound, double upperBound, double tolerance=1e-5, int maxIterations=1000) { var objective = new SimpleObjectiveFunction1D(function); - var algorithm = new GoldenSectionMinimizer(tolerance, maxIterations); - var result = algorithm.FindMinimum(objective, lowerBound, upperBound); + var result = GoldenSectionMinimizer.Minimum(objective, lowerBound, upperBound, tolerance, maxIterations); return result.MinimizingPoint; } @@ -55,8 +54,7 @@ namespace MathNet.Numerics public static Vector OfFunction(Func, double> function, Vector initialGuess, double tolerance=1e-8, int maxIterations=1000) { var objective = ObjectiveFunction.Value(function); - var algorithm = new NelderMeadSimplex(tolerance, maxIterations); - var result = algorithm.FindMinimum(objective, initialGuess); + var result = NelderMeadSimplex.Minimum(objective, initialGuess, tolerance, maxIterations); return result.MinimizingPoint; } @@ -126,11 +124,10 @@ namespace MathNet.Numerics /// Find vector x that minimizes the function f(x) using the Newton algorithm. /// For more options and diagnostics consider to use directly. /// - public static Vector OfFunctionGradientHessian(Func, double> function, Func, Vector> gradient, Func, Matrix> hessian, Vector initialGuess, double tolerance=1e-8, int maxIterations=1000) + public static Vector OfFunctionGradientHessian(Func, double> function, Func, Vector> gradient, Func, Matrix> hessian, Vector initialGuess, double gradientTolerance=1e-8, int maxIterations=1000) { var objective = ObjectiveFunction.GradientHessian(function, gradient, hessian); - var algorithm = new NewtonMinimizer(tolerance, maxIterations); - var result = algorithm.FindMinimum(objective, initialGuess); + var result = NewtonMinimizer.Minimum(objective, initialGuess, gradientTolerance, maxIterations); return result.MinimizingPoint; } @@ -138,11 +135,10 @@ namespace MathNet.Numerics /// Find vector x that minimizes the function f(x) using the Newton algorithm. /// For more options and diagnostics consider to use directly. /// - public static Vector OfFunctionGradientHessian(Func, Tuple, Matrix>> functionGradientHessian, Vector initialGuess, double tolerance=1e-8, int maxIterations=1000) + public static Vector OfFunctionGradientHessian(Func, Tuple, Matrix>> functionGradientHessian, Vector initialGuess, double gradientTolerance=1e-8, int maxIterations=1000) { var objective = ObjectiveFunction.GradientHessian(functionGradientHessian); - var algorithm = new NewtonMinimizer(tolerance, maxIterations); - var result = algorithm.FindMinimum(objective, initialGuess); + var result = NewtonMinimizer.Minimum(objective, initialGuess, gradientTolerance, maxIterations); return result.MinimizingPoint; } } diff --git a/src/Numerics/Optimization/BfgsBMinimizer.cs b/src/Numerics/Optimization/BfgsBMinimizer.cs index f1dd98fa..524f7f08 100644 --- a/src/Numerics/Optimization/BfgsBMinimizer.cs +++ b/src/Numerics/Optimization/BfgsBMinimizer.cs @@ -226,7 +226,7 @@ namespace MathNet.Numerics.Optimization return lineSearchDirection; } - private static Vector ReducedToFull(List reducedMap, Vector reducedVector, Vector fullVector) + static Vector ReducedToFull(List reducedMap, Vector reducedVector, Vector fullVector) { var output = fullVector.Clone(); for (int ii = 0; ii < reducedMap.Count; ++ii) @@ -234,10 +234,10 @@ namespace MathNet.Numerics.Optimization return output; } - private Vector _lowerBound; - private Vector _upperBound; + Vector _lowerBound; + Vector _upperBound; - private static double FindMaxStep(Vector startingPoint, Vector searchDirection, Vector lowerBound, Vector upperBound) + static double FindMaxStep(Vector startingPoint, Vector searchDirection, Vector lowerBound, Vector upperBound) { double maxStep = Double.PositiveInfinity; for (int ii = 0; ii < startingPoint.Count; ++ii) @@ -256,7 +256,7 @@ namespace MathNet.Numerics.Optimization return maxStep; } - private static void CreateReducedData(Vector initialPoint, Vector cauchyPoint, List isFixed, Vector lowerBound, Vector upperBound, Vector gradient, Matrix pseudoHessian, Vector reducedInitialPoint, Vector reducedCauchyPoint, Vector reducedGradient, Matrix reducedHessian, List reducedMap) + static void CreateReducedData(Vector initialPoint, Vector cauchyPoint, List isFixed, Vector lowerBound, Vector upperBound, Vector gradient, Matrix pseudoHessian, Vector reducedInitialPoint, Vector reducedCauchyPoint, Vector reducedGradient, Matrix reducedHessian, List reducedMap) { int ll = 0; for (int ii = 0; ii < lowerBound.Count; ++ii) diff --git a/src/Numerics/Optimization/BfgsMinimizer.cs b/src/Numerics/Optimization/BfgsMinimizer.cs index f16ab494..a1837378 100644 --- a/src/Numerics/Optimization/BfgsMinimizer.cs +++ b/src/Numerics/Optimization/BfgsMinimizer.cs @@ -36,7 +36,7 @@ namespace MathNet.Numerics.Optimization /// /// Broyden–Fletcher–Goldfarb–Shanno (BFGS) algorithm is an iterative method for solving unconstrained nonlinear optimization problems /// - public class BfgsMinimizer : BfgsMinimizerBase + public class BfgsMinimizer : BfgsMinimizerBase, IUnconstrainedMinimizer { /// /// Creates BFGS minimizer diff --git a/src/Numerics/Optimization/BfgsMinimizerBase.cs b/src/Numerics/Optimization/BfgsMinimizerBase.cs index a9b5a975..1917960e 100644 --- a/src/Numerics/Optimization/BfgsMinimizerBase.cs +++ b/src/Numerics/Optimization/BfgsMinimizerBase.cs @@ -50,7 +50,7 @@ namespace MathNet.Numerics.Optimization /// The parameter tolerance /// The funciton progress tolerance /// The maximum number of iterations - public BfgsMinimizerBase(double gradientTolerance, double parameterTolerance, double functionProgressTolerance, int maximumIterations) + protected BfgsMinimizerBase(double gradientTolerance, double parameterTolerance, double functionProgressTolerance, int maximumIterations) { GradientTolerance = gradientTolerance; ParameterTolerance = parameterTolerance; diff --git a/src/Numerics/Optimization/ConjugateGradientMinimizer.cs b/src/Numerics/Optimization/ConjugateGradientMinimizer.cs index 8a7d6d4d..faffa17b 100644 --- a/src/Numerics/Optimization/ConjugateGradientMinimizer.cs +++ b/src/Numerics/Optimization/ConjugateGradientMinimizer.cs @@ -33,7 +33,7 @@ using MathNet.Numerics.Optimization.LineSearch; namespace MathNet.Numerics.Optimization { - public class ConjugateGradientMinimizer + public class ConjugateGradientMinimizer : IUnconstrainedMinimizer { public double GradientTolerance { get; set; } public int MaximumIterations { get; set; } @@ -45,17 +45,26 @@ namespace MathNet.Numerics.Optimization } public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector initialGuess) + { + return Minimum(objective, initialGuess, GradientTolerance, MaximumIterations); + } + + public static MinimizationResult Minimum(IObjectiveFunction objective, Vector initialGuess, double gradientTolerance=1e-8, int maxIterations=1000) { if (!objective.IsGradientSupported) + { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization."); + } objective.EvaluateAt(initialGuess); var gradient = objective.Gradient; ValidateGradient(objective); // Check that we're not already done - if (ExitCriteriaSatisfied(initialGuess, gradient)) + if (gradient.Norm(2.0) < gradientTolerance) + { return new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient); + } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, 1000); @@ -63,7 +72,7 @@ namespace MathNet.Numerics.Optimization // First step var steepestDirection = -gradient; var searchDirection = steepestDirection; - double initialStepSize = 100 * GradientTolerance / (gradient * gradient); + double initialStepSize = 100 * gradientTolerance / (gradient * gradient); LineSearchResult result; try @@ -85,7 +94,7 @@ namespace MathNet.Numerics.Optimization int totalLineSearchSteps = result.Iterations; int iterationsWithNontrivialLineSearch = result.Iterations > 0 ? 0 : 1; int steepestDescentResets = 0; - while (!ExitCriteriaSatisfied(objective.Point, objective.Gradient) && iterations < MaximumIterations) + while (objective.Gradient.Norm(2.0) >= gradientTolerance && iterations < maxIterations) { var previousSteepestDirection = steepestDirection; steepestDirection = -objective.Gradient; @@ -113,32 +122,31 @@ namespace MathNet.Numerics.Optimization iterations += 1; } - if (iterations == MaximumIterations) + if (iterations == maxIterations) { - throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations)); + throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", maxIterations)); } return new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch); } - bool ExitCriteriaSatisfied(Vector candidatePoint, Vector gradient) - { - return gradient.Norm(2.0) < GradientTolerance; - } - - void ValidateGradient(IObjectiveFunctionEvaluation objective) + static void ValidateGradient(IObjectiveFunctionEvaluation objective) { foreach (var x in objective.Gradient) { if (Double.IsNaN(x) || Double.IsInfinity(x)) + { throw new EvaluationException("Non-finite gradient returned.", objective); + } } } - void ValidateObjective(IObjectiveFunctionEvaluation objective) + static void ValidateObjective(IObjectiveFunctionEvaluation objective) { if (Double.IsNaN(objective.Value) || Double.IsInfinity(objective.Value)) + { throw new EvaluationException("Non-finite objective function returned.", objective); + } } } } diff --git a/src/Numerics/Optimization/GoldenSectionMinimizer.cs b/src/Numerics/Optimization/GoldenSectionMinimizer.cs index 43d43016..b784dcc9 100644 --- a/src/Numerics/Optimization/GoldenSectionMinimizer.cs +++ b/src/Numerics/Optimization/GoldenSectionMinimizer.cs @@ -39,7 +39,7 @@ namespace MathNet.Numerics.Optimization public double LowerExpansionFactor { get; set; } public double UpperExpansionFactor { get; set; } - public GoldenSectionMinimizer(double xTolerance = 1e-5, int maxIterations = 1000, int maxExpansionSteps = 10, double lowerExpansionFactor = 2.0, double upperExpansionFactor = 2.0) + public GoldenSectionMinimizer(double xTolerance=1e-5, int maxIterations=1000, int maxExpansionSteps=10, double lowerExpansionFactor=2.0, double upperExpansionFactor=2.0) { XTolerance = xTolerance; MaximumIterations = maxIterations; @@ -53,7 +53,7 @@ namespace MathNet.Numerics.Optimization return Minimum(objective, lowerBound, upperBound, XTolerance, MaximumIterations, MaximumExpansionSteps, LowerExpansionFactor, UpperExpansionFactor); } - public static MinimizationResult1D Minimum(IObjectiveFunction1D objective, double lowerBound, double upperBound, double xTolerance = 1e-5, int maxIterations = 1000, int maxExpansionSteps = 10, double lowerExpansionFactor = 2.0, double upperExpansionFactor = 2.0) + public static MinimizationResult1D Minimum(IObjectiveFunction1D objective, double lowerBound, double upperBound, double xTolerance=1e-5, int maxIterations=1000, int maxExpansionSteps=10, double lowerExpansionFactor=2.0, double upperExpansionFactor=2.0) { if (upperBound <= lowerBound) { diff --git a/src/Numerics/Optimization/NelderMeadSimplex.cs b/src/Numerics/Optimization/NelderMeadSimplex.cs index 8f1c2d58..2aebb9dc 100644 --- a/src/Numerics/Optimization/NelderMeadSimplex.cs +++ b/src/Numerics/Optimization/NelderMeadSimplex.cs @@ -41,7 +41,7 @@ namespace MathNet.Numerics.Optimization /// or /// https://en.wikipedia.org/wiki/Nelder%E2%80%93Mead_method /// - public sealed class NelderMeadSimplex + public sealed class NelderMeadSimplex : IUnconstrainedMinimizer { static readonly double JITTER = 1e-10d; // a small value used to protect against floating point noise @@ -87,7 +87,7 @@ namespace MathNet.Numerics.Optimization /// The objective function, no gradient or hessian needed /// The intial guess /// The minimum point - public static MinimizationResult Minimum(IObjectiveFunction objectiveFunction, Vector initialGuess, double convergenceTolerance, int maximumIterations=1000) + public static MinimizationResult Minimum(IObjectiveFunction objectiveFunction, Vector initialGuess, double convergenceTolerance=1e-8, int maximumIterations=1000) { var initalPertubation = new LinearAlgebra.Double.DenseVector(initialGuess.Count); for (int i = 0; i < initialGuess.Count; i++) @@ -104,7 +104,7 @@ namespace MathNet.Numerics.Optimization /// The intial guess /// The inital pertubation /// The minimum point - public static MinimizationResult Minimum(IObjectiveFunction objectiveFunction, Vector initialGuess, Vector initalPertubation, double convergenceTolerance, int maximumIterations=1000) + public static MinimizationResult Minimum(IObjectiveFunction objectiveFunction, Vector initialGuess, Vector initalPertubation, double convergenceTolerance=1e-8, int maximumIterations=1000) { // confirm that we are in a position to commence if (objectiveFunction == null) diff --git a/src/Numerics/Optimization/NewtonMinimizer.cs b/src/Numerics/Optimization/NewtonMinimizer.cs index eac9a003..571981a6 100644 --- a/src/Numerics/Optimization/NewtonMinimizer.cs +++ b/src/Numerics/Optimization/NewtonMinimizer.cs @@ -33,7 +33,7 @@ using MathNet.Numerics.Optimization.LineSearch; namespace MathNet.Numerics.Optimization { - public sealed class NewtonMinimizer + public sealed class NewtonMinimizer : IUnconstrainedMinimizer { public double GradientTolerance { get; set; } public int MaximumIterations { get; set; } @@ -51,7 +51,7 @@ namespace MathNet.Numerics.Optimization return Minimum(objective, initialGuess, GradientTolerance, MaximumIterations, UseLineSearch); } - public static MinimizationResult Minimum(IObjectiveFunction objective, Vector initialGuess, double gradientTolerance, int maxIterations=1000, bool useLineSearch = false) + public static MinimizationResult Minimum(IObjectiveFunction objective, Vector initialGuess, double gradientTolerance=1e-8, int maxIterations=1000, bool useLineSearch=false) { if (!objective.IsGradientSupported) { diff --git a/src/UnitTests/OptimizationTests/ConjugateGradientMinimizerTests.cs b/src/UnitTests/OptimizationTests/ConjugateGradientMinimizerTests.cs index adbcf570..ddd6d5b2 100644 --- a/src/UnitTests/OptimizationTests/ConjugateGradientMinimizerTests.cs +++ b/src/UnitTests/OptimizationTests/ConjugateGradientMinimizerTests.cs @@ -109,9 +109,8 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests public void Mgh_Tests(TestFunctions.TestCase test_case) { var obj = new MghObjectiveFunction(test_case.Function, true, true); - var solver = new ConjugateGradientMinimizer(1e-8, 1000); - var result = solver.FindMinimum(obj, test_case.InitialGuess); + var result = ConjugateGradientMinimizer.Minimum(obj, test_case.InitialGuess, 1e-8, 1000); if (test_case.MinimizingPoint != null) {