Erik Ovegard
10 years ago
14 changed files with 616 additions and 438 deletions
@ -0,0 +1,158 @@ |
|||
// <copyright file="BfgsTest.cs" company="Math.NET">
|
|||
// Math.NET Numerics, part of the Math.NET Project
|
|||
// http://numerics.mathdotnet.com
|
|||
// http://github.com/mathnet/mathnet-numerics
|
|||
// http://mathnetnumerics.codeplex.com
|
|||
//
|
|||
// Copyright (c) 2009-2016 Math.NET
|
|||
//
|
|||
// Permission is hereby granted, free of charge, to any person
|
|||
// obtaining a copy of this software and associated documentation
|
|||
// files (the "Software"), to deal in the Software without
|
|||
// restriction, including without limitation the rights to use,
|
|||
// copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|||
// copies of the Software, and to permit persons to whom the
|
|||
// Software is furnished to do so, subject to the following
|
|||
// conditions:
|
|||
//
|
|||
// The above copyright notice and this permission notice shall be
|
|||
// included in all copies or substantial portions of the Software.
|
|||
//
|
|||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|||
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|||
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|||
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|||
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
|||
// </copyright>
|
|||
|
|||
using MathNet.Numerics.LinearAlgebra; |
|||
using MathNet.Numerics.LinearAlgebra.Double; |
|||
using MathNet.Numerics.Optimization.LineSearch; |
|||
using System; |
|||
|
|||
namespace MathNet.Numerics.Optimization |
|||
{ |
|||
public abstract class BfgsMinimizerBase |
|||
{ |
|||
public double GradientTolerance { get; set; } |
|||
public double ParameterTolerance { get; set; } |
|||
public double FunctionProgressTolerance { get; set; } |
|||
public int MaximumIterations { get; set; } |
|||
|
|||
protected const double VerySmall = 1e-15; |
|||
|
|||
/// <summary>
|
|||
/// Creates a base class for BFGS minimization
|
|||
/// </summary>
|
|||
/// <param name="gradientTolerance">The gradient tolerance</param>
|
|||
/// <param name="parameterTolerance">The parameter tolerance</param>
|
|||
/// <param name="functionProgressTolerance">The funciton progress tolerance</param>
|
|||
/// <param name="maximumIterations">The maximum number of iterations</param>
|
|||
public BfgsMinimizerBase(double gradientTolerance, double parameterTolerance, double functionProgressTolerance, int maximumIterations) |
|||
{ |
|||
GradientTolerance = gradientTolerance; |
|||
ParameterTolerance = parameterTolerance; |
|||
FunctionProgressTolerance = functionProgressTolerance; |
|||
MaximumIterations = maximumIterations; |
|||
} |
|||
|
|||
protected MinimizationResult.ExitCondition ExitCriteriaSatisfied(IObjectiveFunction candidatePoint, IObjectiveFunction lastPoint, int iterations) |
|||
{ |
|||
Vector<double> relGrad = new DenseVector(candidatePoint.Point.Count); |
|||
double relativeGradient = 0.0; |
|||
double normalizer = Math.Max(Math.Abs(candidatePoint.Value), 1.0); |
|||
for (int ii = 0; ii < relGrad.Count; ++ii) |
|||
{ |
|||
double projectedGradient = GetProjectedGradient(candidatePoint, ii); |
|||
|
|||
double tmp = projectedGradient * |
|||
Math.Max(Math.Abs(candidatePoint.Point[ii]), 1.0) / normalizer; |
|||
relativeGradient = Math.Max(relativeGradient, Math.Abs(tmp)); |
|||
} |
|||
if (relativeGradient < GradientTolerance) |
|||
{ |
|||
return MinimizationResult.ExitCondition.RelativeGradient; |
|||
} |
|||
|
|||
if (lastPoint != null) |
|||
{ |
|||
double mostProgress = 0.0; |
|||
for (int ii = 0; ii < candidatePoint.Point.Count; ++ii) |
|||
{ |
|||
var tmp = Math.Abs(candidatePoint.Point[ii] - lastPoint.Point[ii]) / |
|||
Math.Max(Math.Abs(lastPoint.Point[ii]), 1.0); |
|||
mostProgress = Math.Max(mostProgress, tmp); |
|||
} |
|||
if (mostProgress < ParameterTolerance) |
|||
{ |
|||
return MinimizationResult.ExitCondition.LackOfProgress; |
|||
} |
|||
|
|||
double functionChange = candidatePoint.Value - lastPoint.Value; |
|||
if (iterations > 500 && functionChange < 0 && Math.Abs(functionChange) < FunctionProgressTolerance) |
|||
return MinimizationResult.ExitCondition.LackOfProgress; |
|||
} |
|||
|
|||
return MinimizationResult.ExitCondition.None; |
|||
} |
|||
|
|||
protected virtual double GetProjectedGradient(IObjectiveFunction candidatePoint, int ii) |
|||
{ |
|||
return candidatePoint.Gradient[ii]; |
|||
} |
|||
|
|||
protected void ValidateGradientAndObjective(IObjectiveFunction eval) |
|||
{ |
|||
foreach (var x in eval.Gradient) |
|||
{ |
|||
if (Double.IsNaN(x) || Double.IsInfinity(x)) |
|||
throw new EvaluationException("Non-finite gradient returned.", eval); |
|||
} |
|||
if (Double.IsNaN(eval.Value) || Double.IsInfinity(eval.Value)) |
|||
throw new EvaluationException("Non-finite objective function returned.", eval); |
|||
} |
|||
|
|||
protected int DoBfgsUpdate(ref MinimizationResult.ExitCondition currentExitCondition, WolfeLineSearch lineSearcher, ref Matrix<double> inversePseudoHessian, ref Vector<double> lineSearchDirection, ref IObjectiveFunction previousPoint, ref LineSearchResult lineSearchResult, ref IObjectiveFunction candidate, ref Vector<double> step, ref int totalLineSearchSteps, ref int iterationsWithNontrivialLineSearch) |
|||
{ |
|||
int iterations; |
|||
for (iterations = 1; iterations < MaximumIterations; ++iterations) |
|||
{ |
|||
double startingStepSize; |
|||
double maxLineSearchStep; |
|||
lineSearchDirection = CalculateSearchDirection(ref inversePseudoHessian, out maxLineSearchStep, out startingStepSize, previousPoint, candidate, step); |
|||
|
|||
try |
|||
{ |
|||
lineSearchResult = lineSearcher.FindConformingStep(candidate, lineSearchDirection, startingStepSize, maxLineSearchStep); |
|||
} |
|||
catch (Exception e) |
|||
{ |
|||
throw new InnerOptimizationException("Line search failed.", e); |
|||
} |
|||
|
|||
iterationsWithNontrivialLineSearch += lineSearchResult.Iterations > 0 ? 1 : 0; |
|||
totalLineSearchSteps += lineSearchResult.Iterations; |
|||
|
|||
step = lineSearchResult.FunctionInfoAtMinimum.Point - candidate.Point; |
|||
previousPoint = candidate; |
|||
candidate = lineSearchResult.FunctionInfoAtMinimum; |
|||
|
|||
currentExitCondition = ExitCriteriaSatisfied(candidate, previousPoint, iterations); |
|||
if (currentExitCondition != MinimizationResult.ExitCondition.None) |
|||
break; |
|||
} |
|||
|
|||
return iterations; |
|||
} |
|||
|
|||
protected abstract Vector<double> CalculateSearchDirection(ref Matrix<double> inversePseudoHessian, |
|||
out double maxLineSearchStep, |
|||
out double startingStepSize, |
|||
IObjectiveFunction previousPoint, |
|||
IObjectiveFunction candidate, |
|||
Vector<double> step); |
|||
} |
|||
} |
|||
@ -1,111 +0,0 @@ |
|||
// <copyright file="WolfeRule.cs" company="Math.NET">
|
|||
// Math.NET Numerics, part of the Math.NET Project
|
|||
// http://numerics.mathdotnet.com
|
|||
// http://github.com/mathnet/mathnet-numerics
|
|||
// http://mathnetnumerics.codeplex.com
|
|||
//
|
|||
// Copyright (c) 2009-2015 Math.NET
|
|||
//
|
|||
// Permission is hereby granted, free of charge, to any person
|
|||
// obtaining a copy of this software and associated documentation
|
|||
// files (the "Software"), to deal in the Software without
|
|||
// restriction, including without limitation the rights to use,
|
|||
// copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|||
// copies of the Software, and to permit persons to whom the
|
|||
// Software is furnished to do so, subject to the following
|
|||
// conditions:
|
|||
//
|
|||
// The above copyright notice and this permission notice shall be
|
|||
// included in all copies or substantial portions of the Software.
|
|||
//
|
|||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|||
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|||
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|||
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|||
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
|||
// </copyright>
|
|||
|
|||
using System; |
|||
using MathNet.Numerics.LinearAlgebra; |
|||
|
|||
namespace MathNet.Numerics.Optimization.LineSearch |
|||
{ |
|||
/// <summary>
|
|||
/// Performs an inexact line search. This is used as a part of quasi-Newton optimization methods to figure
|
|||
/// out how far to move along a certain gradient.
|
|||
/// See http://en.wikipedia.org/wiki/Wolfe_conditions
|
|||
/// Inspired by implementation: https://github.com/PatWie/CppNumericalSolvers/blob/master/src/linesearch/WolfeRule.h
|
|||
/// </summary>
|
|||
internal static class WolfeRule |
|||
{ |
|||
/// <summary>
|
|||
/// Searches along a line to satisfy the Wolfe conditions (inexact search for minimum)
|
|||
/// </summary>
|
|||
/// <param name="x0">Starting point of search</param>
|
|||
/// <param name="z">Search direction</param>
|
|||
/// <param name="functionValue">Evaluates the function being minimized</param>
|
|||
/// <param name="functionGradient">Evaluates the gradient of the function</param>
|
|||
/// <param name="alphaInit">Initial value for the coefficient of z (distance to travel in z direction)</param>
|
|||
/// <returns></returns>
|
|||
public static double LineSearch( |
|||
Vector<double> x0, |
|||
Vector<double> z, |
|||
Func<Vector<double>, double> functionValue, |
|||
Func<Vector<double>, Vector<double>> functionGradient, |
|||
float alphaInit = 1) |
|||
{ |
|||
Vector<double> x = x0; |
|||
|
|||
// evaluate phi(0)
|
|||
double phi0 = functionValue(x0); |
|||
|
|||
// evaluate phi'(0)
|
|||
Vector<double> grad = functionGradient(x); |
|||
double phi0_dash = z * grad; |
|||
|
|||
double alpha = alphaInit; |
|||
|
|||
bool decrease_direction = true; |
|||
|
|||
// 200 guesses max
|
|||
for (int iter = 0; iter < 200; ++iter) { |
|||
|
|||
// new guess for phi(alpha)
|
|||
Vector<double> x_candidate = x + alpha * z; |
|||
double phi = functionValue(x_candidate); |
|||
|
|||
// decrease condition invalid --> shrink interval
|
|||
if (phi > phi0 + 0.0001 * alpha * phi0_dash) |
|||
{ |
|||
alpha *= 0.5; |
|||
decrease_direction = false; |
|||
} |
|||
else |
|||
{ |
|||
// valid decrease --> test strong wolfe condition
|
|||
Vector<double> grad2 = functionGradient(x_candidate); |
|||
double phi_dash = z * grad2; |
|||
|
|||
// curvature condition invalid ?
|
|||
if ((phi_dash < 0.9 * phi0_dash) || !decrease_direction) { |
|||
// increase interval
|
|||
alpha *= 4.0; |
|||
} |
|||
else { |
|||
// both condition are valid --> we are happy
|
|||
x = x_candidate; |
|||
grad = grad2; |
|||
phi0 = phi; |
|||
return alpha; |
|||
} |
|||
} |
|||
} |
|||
|
|||
|
|||
return alpha; |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue