From aca34e13b977dc6836c4dc680f8ec3440772740c Mon Sep 17 00:00:00 2001 From: diluculo Date: Fri, 21 Dec 2018 19:37:06 +0900 Subject: [PATCH 01/13] Optimization: added Levenberg-Marquardt and trust region dogleg minimizer. --- .../NonLinearCurveFittingTests.cs | 404 ++++++++++ src/Numerics/Optimization/ExitCondition.cs | 6 +- src/Numerics/Optimization/IObjectiveModel.cs | 70 ++ .../LevenbergMarquardtMinimizer.cs | 263 +++++++ .../Optimization/ModelMinimizationResult.cs | 52 ++ src/Numerics/Optimization/ObjectiveModel.cs | 40 + .../ObjectiveModels/FittingObjectiveModel.cs | 729 ++++++++++++++++++ .../TrustRegionDogLegMinimizer.cs | 322 ++++++++ 8 files changed, 1885 insertions(+), 1 deletion(-) create mode 100644 src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs create mode 100644 src/Numerics/Optimization/IObjectiveModel.cs create mode 100644 src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs create mode 100644 src/Numerics/Optimization/ModelMinimizationResult.cs create mode 100644 src/Numerics/Optimization/ObjectiveModel.cs create mode 100644 src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs create mode 100644 src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs new file mode 100644 index 00000000..fbbd3448 --- /dev/null +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -0,0 +1,404 @@ +using MathNet.Numerics.LinearAlgebra; +using MathNet.Numerics.LinearAlgebra.Double; +using MathNet.Numerics.Optimization; +using NUnit.Framework; +using System; + +namespace MathNet.Numerics.UnitTests.OptimizationTests +{ + [TestFixture] + public class NonLinearCurveFittingTests + { + // model: Rosenbrock + // f(x; a, b) = (1 - a)^2 + 100*(b - a^2)^2 + // derivatives: + // df/da = 400*a^3 - 400*a*b + 2*a - 2 + // df/db = 200*(b - a^2) + // best fitted parameters: + // a = 1 + // b = 1 + private double RosenbrockModel(Vector p, double x) + { + var y = Math.Pow(1.0 - p[0], 2) + 100.0 * Math.Pow(p[1] - p[0] * p[0], 2); + return y; + } + private Vector RosenbrockPrime(Vector p, double x) + { + var prime = Vector.Build.Dense(p.Count); + prime[0] = 400.0 * p[0] * p[0] * p[0] - 400.0 * p[0] * p[1] + 2.0 * p[0] - 2.0; + prime[1] = 200.0 * (p[1] - p[0] * p[0]); + return prime; + } + private Vector RosenbrockX = Vector.Build.Dense(2); + private Vector RosenbrockY = Vector.Build.Dense(2); + private Vector RosenbrockPbest = new DenseVector(new double[] { 1.0, 1.0 }); + + private Vector RosenbrockStart1 = new DenseVector(new double[] { -1.2, 1.0 }); + private Vector RosebbrockLowerBound = new DenseVector(new double[] { -5.0, -5.0 }); + private Vector RosenbrockUpperBound = new DenseVector(new double[] { 5.0, 5.0 }); + + [Test] + public void LMDER_FindMinimum_Rosenbrock_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); + + var result = solver.FindMinimum(obj, RosenbrockStart1); + + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + } + + [Test] + public void LMDIF_FindMinimum_Rosenbrock_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder:2); + var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); + + var result = solver.FindMinimum(obj, RosenbrockStart1); + + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + } + + [Test] + public void LMDER_FindMinimum_Rosenbrock_BoxConstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY, + lowerBound : RosebbrockLowerBound, upperBound : RosenbrockUpperBound); + var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); + + var result = solver.FindMinimum(obj, RosenbrockStart1); + + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + } + + [Test] + public void LMDIF_FindMinimum_Rosenbrock_BoxConstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, + lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound, + accuracyOrder: 2); + var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); + + var result = solver.FindMinimum(obj, RosenbrockStart1); + + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + } + + [Test] + public void TRLMDER_FindMinimum_Rosenbrock_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); + + var result = solver.FindMinimum(obj, RosenbrockStart1); + + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 1); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 1); + } + + [Test] + public void TRLMDIF_FindMinimum_Rosenbrock_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); + + var result = solver.FindMinimum(obj, RosenbrockStart1); + + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 1); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 1); + } + + // model: Rat43 (https://www.itl.nist.gov/div898/strd/nls/data/ratkowsky3.shtml) + // f(x; a, b, c, d) = a / ((1 + exp(b - c * x))^(1 / d)) + // best fitted parameters: + // a = 6.9964151270E+02 +/- 1.6302297817E+01 + // b = 5.2771253025E+00 +/- 2.0828735829E+00 + // c = 7.5962938329E-01 +/- 1.9566123451E-01 + // d = 1.2792483859E+00 +/- 6.8761936385E-01 + private double Rat43Model(Vector p, double x) + { + var y = p[0] / Math.Pow(1.0 + Math.Exp(p[1] - p[2] * x), 1.0 / p[3]); + return y; + } + private Vector Rat43X = new DenseVector(new double[] { + 1.00, 2.00, 3.00, 4.00, 5.00, 6.00, 7.00, 8.00, 9.00, 10.00, + 11.00, 12.00, 13.00, 14.00, 15.00 + }); + private Vector Rat43Y = new DenseVector(new double[] { + 16.08, 33.83, 65.80, 97.20, 191.55, 326.20, 386.87, 520.53, 590.03, 651.92, + 724.93, 699.56, 689.96, 637.56, 717.41 + }); + private Vector Rat43Pbest = new DenseVector(new double[] { + 6.9964151270E+02, 5.2771253025E+00, 7.5962938329E-01, 1.2792483859E+00 + }); + private Vector Rat43Pstd = new DenseVector(new double[]{ + 1.6302297817E+01, 2.0828735829E+00, 1.9566123451E-01, 6.8761936385E-01 + }); + + private Vector Rat43Start1 = new DenseVector(new double[] { 100, 10, 1, 1 }); + private Vector Rat43Start2 = new DenseVector(new double[] { 700, 5, 0.75, 1.3 }); + + [Test] + public void LMDIF_FindMinimum_Rat43_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, Rat43Start1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(Rat43Pstd[i], result.StandardErrors[i], 6); + } + } + + [Test] + public void TRLMDIF_FindMinimum_Rat43_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var solver = new TrustRegionDogLegMinimizer(); + + var result = solver.FindMinimum(obj, Rat43Start2); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.BestFitParameters[i], 2); + AssertHelpers.AlmostEqualRelative(Rat43Pstd[i], result.StandardErrors[i], 2); + } + } + + // model: BoxBod (https://www.itl.nist.gov/div898/strd/nls/data/boxbod.shtml) + // f(x; a, b) = a*(1 - exp(-b*x)) + // derivatives: + // df/da = 1 - exp(-b*x) + // df/db = a*x*exp(-b*x) + // best fitted parameters: + // a = 2.1380940889E+02 +/- 1.2354515176E+01 + // b = 5.4723748542E-01 +/- 1.0455993237E-01 + private double BoxBodModel(Vector p, double x) + { + var y = p[0] * (1.0 - Math.Exp(-p[1] * x)); + return y; + } + private Vector BoxBodPrime(Vector p, double x) + { + var prime = Vector.Build.Dense(p.Count); + prime[0] = 1.0 - Math.Exp(-p[1] * x); + prime[1] = p[0] * x * Math.Exp(-p[1] * x); + return prime; + } + private Vector BoxBodX = new DenseVector(new double[] { 1, 2, 3, 5, 7, 10 }); + private Vector BoxBodY = new DenseVector(new double[] { 109, 149, 149, 191, 213, 224 }); + private Vector BoxBodPbest = new DenseVector(new double[] { 2.1380940889E+02, 5.4723748542E-01 }); + private Vector BoxBodPstd = new DenseVector(new double[] { 1.2354515176E+01, 1.0455993237E-01 }); + + private Vector BoxBodStart1 = new DenseVector(new double[] { 1.0, 1.0 }); + private Vector BoxBodStart2 = new DenseVector(new double[] { 100.0, 0.75 }); + private Vector BoxBodLowerBound = new DenseVector(new double[] { 0, 0 }); + private Vector BoxBodUpperBound = new DenseVector(new double[] { 500.0, 10 }); + private Vector BoxBodScales = new DenseVector(new double[] { 100.0, 1 }); + + [Test] + public void LMDER_FindMinimum_BoxBod_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, BoxBodStart1); + + AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); + + AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + } + + [Test] + public void LMDIF_FindMinimum_BoxBod_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder:6); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, BoxBodStart1); + + AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); + + AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + } + + [Test] + public void LMDER_FindMinimum_BoxBod_BoxConstrained() + { + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, BoxBodStart1); + + AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); + + AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + } + + [Test] + public void LMDIF_FindMinimum_BoxBod_BoxConstrained() + { + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, + lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound, + accuracyOrder: 6); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, BoxBodStart1); + + AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); + + AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + } + + [Test] + public void TRLMDIF_FindMinimum_BoxBod_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); + var solver = new TrustRegionDogLegMinimizer(); + + var result = solver.FindMinimum(obj, BoxBodStart1); + + AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 3); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 3); + + AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 3); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 3); + } + + // model : Thurber (https://www.itl.nist.gov/div898/strd/nls/data/thurber.shtml) + // f(x; b1 ... b7) = (b1 + b2*x + b3*x^2 + b4*x^3) / (1 + b5*x + b6*x^2 + b7*x^3) + // derivatives: + // df/db1 = 1/(b5*x + b6*x^2 + b7*x^3 + 1) + // df/db2 = x/(b5*x + b6*x^2 + b7*x^3 + 1) + // df/db3 = x^2/(b5*x + b6*x^2 + b7*x^3 + 1) + // df/db4 = x^3/(b5*x + b6*x^2 + b7*x^3 + 1) + // df/db5 = -(x*(b1 + x*(b2 + x*(b3 + b4*x))))/(b5*x + b6*x^2 + b7*x^3 + 1)^2 + // df/db6 = -(x^2*(b1 + x*(b2 + x*(b3 + b4*x))))/(b5*x + b6*x^2 + b7*x^3 + 1)^2 + // df/db7 = -(x^3*(b1 + x*(b2 + x*(b3 + b4*x))))/(b5*x + b6*x^2 + b7*x^3 + 1)^2 + // best fitted parameters: + // b1 = 1.2881396800E+03 +/- 4.6647963344E+00 + // b2 = 1.4910792535E+03 +/- 3.9571156086E+01 + // b3 = 5.8323836877E+02 +/- 2.8698696102E+01 + // b4 = 7.5416644291E+01 +/- 5.5675370270E+00 + // b5 = 9.6629502864E-01 +/- 3.1333340687E-02 + // b6 = 3.9797285797E-01 +/- 1.4984928198E-02 + // b7 = 4.9727297349E-02 +/- 6.5842344623E-03 + private double ThurberModel(Vector p, double x) + { + var xSq = x * x; + var xCb = xSq * x; + + var y = (p[0] + p[1] * x + p[2] * xSq + p[3] * xCb) + / (1 + p[4] * x + p[5] * xSq + p[6] * xCb); + return y; + } + private Vector ThurberPrime(Vector p, double x) + { + var prime = Vector.Build.Dense(p.Count); + + var xSq = x * x; + var xCb = xSq * x; + var num = (p[0] + x * (p[1] + x * (p[2] + p[3] * x))); + var den = (p[4] * x + p[5] * xSq + p[6] * xCb + 1.0); + var denSq = den * den; + + prime[0] = 1.0 / den; + prime[1] = x / den; + prime[2] = xSq / den; + prime[3] = xCb / den; + prime[4] = -(x * num) / denSq; + prime[5] = -(xSq * num) / denSq; + prime[6] = -(xCb * num) / denSq; + return prime; + } + private Vector ThurberX = new DenseVector(new double[] { + -3.067, -2.981, -2.921, -2.912, -2.84, + -2.797, -2.702, -2.699, -2.633, -2.481, + -2.363, -2.322, -1.501, -1.460, -1.274, + -1.212, -1.100, -1.046, -0.915, -0.714, + -0.566, -0.545, -0.400, -0.309, -0.109, + -0.103, 0.01, 0.119, 0.377, 0.79, + 0.963, 1.006, 1.115, 1.572, 1.841, + 2.047, 2.2}); + private Vector ThurberY = new DenseVector(new double[] { + 80.574, 084.248, 087.264, 087.195, 089.076, + 089.608, 089.868, 090.101, 092.405, 095.854, + 100.696, 101.060, 401.672, 390.724, 567.534, + 635.316, 733.054, 759.087, 894.206, 990.785, + 1090.109, 1080.914, 1122.643, 1178.351, 1260.531, + 1273.514, 1288.339, 1327.543, 1353.863, 1414.509, + 1425.208, 1421.384, 1442.962, 1464.350, 1468.705, + 1447.894, 1457.628}); + private Vector ThurberPbest = new DenseVector(new double[] { + 1.2881396800E+03, 1.4910792535E+03, 5.8323836877E+02, 7.5416644291E+01, 9.6629502864E-01, + 3.9797285797E-01, 4.9727297349E-02 }); + private Vector ThurberPstd = new DenseVector(new double[] { + 4.6647963344E+00, 3.9571156086E+01, 2.8698696102E+01, 5.5675370270E+00, 3.1333340687E-02, + 1.4984928198E-02, 6.5842344623E-03 }); + private Vector ThurberInitialGuess = new DenseVector(new double[] { 1000.0, 1000.0, 400.0, 40.0, 0.7, 0.3, 0.03 }); + private Vector ThurberScales = new DenseVector(new double[7] { 1000, 1000, 400, 40, 0.7, 0.3, 0.03 }); + + [Test] + public void LMDER_FindMinimum_Thurber_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberPrime, ThurberX, ThurberY); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, ThurberInitialGuess); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 6); + } + } + + [Test] + public void LMDIF_FindMinimum_Thurber_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var solver = new LevenbergMarquardtMinimizer(); + + var result = solver.FindMinimum(obj, ThurberInitialGuess); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 6); + } + } + + [Test] + public void TRLMDIF_FindMinimum_Thurber_Scaled() + { + var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, + scales: ThurberScales, + accuracyOrder: 6); + var solver = new TrustRegionDogLegMinimizer(); + + var result = solver.FindMinimum(obj, ThurberInitialGuess); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 3); + } + } + } +} diff --git a/src/Numerics/Optimization/ExitCondition.cs b/src/Numerics/Optimization/ExitCondition.cs index 90f30714..dc83feb4 100644 --- a/src/Numerics/Optimization/ExitCondition.cs +++ b/src/Numerics/Optimization/ExitCondition.cs @@ -32,12 +32,16 @@ namespace MathNet.Numerics.Optimization public enum ExitCondition { None, + InvalidValues, + ExceedIterations, + RelativePoints, RelativeGradient, LackOfProgress, AbsoluteGradient, WeakWolfeCriteria, BoundTolerance, StrongWolfeCriteria, - Converged + Converged, + ManuallyStopped } } diff --git a/src/Numerics/Optimization/IObjectiveModel.cs b/src/Numerics/Optimization/IObjectiveModel.cs new file mode 100644 index 00000000..45f94409 --- /dev/null +++ b/src/Numerics/Optimization/IObjectiveModel.cs @@ -0,0 +1,70 @@ +using MathNet.Numerics.LinearAlgebra; + +namespace MathNet.Numerics.Optimization +{ + public interface IObjectiveModelEvaluation + { + IObjectiveModel CreateNew(); + + /// + /// Get the y-values of the fitted model that correspond to the independent values. + /// + Vector Values { get; } + + /// + /// Get the values of the parameters. + /// + Vector Parameters { get; } + + /// + /// Get the residual sum of squares. + /// + double Residue { get; } + + /// + /// Get the Jacobian matrix, J(x; p) = df(x; p)/dp. + /// + Matrix Jacobian { get; } + /// + /// Get the Gradient vector. G = J'(y - f(x; p)) + /// + Vector Gradient { get; } + /// + /// Get the approximated Hessian matrix. H = J'J + /// + Matrix Hessian { get; } + /// + /// Get the covariance matrix. + /// + Matrix Covariance { get; } + + /// + /// Get the number of calls to function. + /// + int FunctionEvaluations { get; } + /// + /// Get the number of calls to jacobian. + /// + int JacobianEvaluations { get; } + + /// + /// Get the degree of freedom. + /// + int DegreeOfFreedom { get; } + + /// + /// Get whether or not the analytical jacobian is supported. + /// + bool IsJacobianSupported { get; } + } + + public interface IObjectiveModel : IObjectiveModelEvaluation + { + void EvaluateFunction(Vector parameters); + void EvaluateJacobian(Vector parameters); + void EvaluateCovariance(Vector parameters); + + /// Create a new independent copy of this objective function, evaluated at the same point. + IObjectiveModel Fork(); + } +} diff --git a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs new file mode 100644 index 00000000..681eb3dd --- /dev/null +++ b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs @@ -0,0 +1,263 @@ +using MathNet.Numerics.LinearAlgebra; +using System; +using System.Linq; + +namespace MathNet.Numerics.Optimization +{ + public sealed class LevenbergMarquardtMinimizer + { + #region Tolerances and options + + /// + /// The scale factor for initial mu + /// + public static double InitialMu { get; set; } + + /// + /// The stopping threshold for infinity norm of the gradient. + /// + public static double GradientTolerance { get; set; } + + /// + /// The stopping threshold for L2 norm of the change of the parameters. + /// + public static double StepTolerance { get; set; } + + /// + /// The stopping threshold for the function value or L2 norm of the residuals. + /// + public static double FunctionTolerance { get; set; } + + /// + /// The maximum number of iterations. + /// + public int MaximumIterations { get; set; } + + #endregion Tolerances and options + + public LevenbergMarquardtMinimizer(double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + { + InitialMu = initialMu; + GradientTolerance = gradientTolerance; + StepTolerance = stepTolerance; + FunctionTolerance = functionTolerance; + MaximumIterations = maximumIterations; + } + + public ModelMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + { + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + return Minimum(objective, initialGuess, InitialMu, FunctionTolerance, GradientTolerance, StepTolerance, MaximumIterations); + } + + public ModelMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + { + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + return Minimum(objective, CreateVector.DenseOfArray(initialGuess), InitialMu, GradientTolerance, StepTolerance, FunctionTolerance, MaximumIterations); + } + + /// + /// Non-linear least square fitting by the Levenberg-Marduardt algorithm. + /// + /// The objective function, including model, observations, and parameter bounds. + /// The initial guess values. + /// The initial damping parameter of mu. + /// The stopping threshold for infinity norm of the gradient vector. + /// The stopping threshold for L2 norm of the change of parameters. + /// The stopping threshold for L2 norm of the residuals. + /// The max iterations. + /// The result of the Levenberg-Marquardt minimization + public static ModelMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + { + // Non-linear least square fitting by the Levenberg-Marduardt algorithm. + // + // Levenberg-Marquardt is finding the minimum of a function F(p) that is a sum of squares of nonlinear functions. + // + // For given datum pair (x, y), uncertainties σ (or weighting W = 1 / σ^2) and model function f = f(x; p), + // let's find the parameters of the model so that the sum of the quares of the deviations is minimized. + // + // F(p) = 1/2 * ∑{ Wi * (yi - f(xi; p))^2 } + // pbest = argmin F(p) + // + // We will use the following terms: + // Weighting W is the diagonal matrix and can be decomposed as LL', so L = 1/σ + // Residuals, R = L(y - f(x; p)) + // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) + // Jacobian J = df(x; p)/dp + // Gradient g = J'W(y − f(x; p)) = J'LR + // Approximated Hessian H = J'WJ + // + // The Levenberg-Marquardt algorithm is summarized as follows: + // initially let μ = τ * max(diag(J'WJ)). + // repeat + // solve linear equations: (J'WJ + μI)ΔP = J'R + // let ρ = (||R||^2 - ||Rnew||^2) / (Δp'(μΔp + J'R)). + // if ρ > ε, P = P + ΔP; μ = μ * max(1/3, 1 - (2ρ - 1)^3); ν = 2; + // otherwise μ = μ*ν; ν = 2*ν; + // + // References: + // [1]. Madsen, K., H. B. Nielsen, and O. Tingleff. + // "Methods for Non-Linear Least Squares Problems. Technical University of Denmark, 2004. Lecture notes." (2004). + // Available Online from: http://orbit.dtu.dk/files/2721358/imm3215.pdf + // [2]. Gavin, Henri. + // "The Levenberg-Marquardt method for nonlinear least squares curve-fitting problems." + // Department of Civil and Environmental Engineering, Duke University (2017): 1-19. + // Availble Online from: http://people.duke.edu/~hpgavin/ce281/lm.pdf + + if (objective == null) + throw new ArgumentNullException("objective"); + + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + ExitCondition exitCondition = ExitCondition.None; + + // First, calculate function values and setup variables + objective.EvaluateFunction(initialGuess); + var P = objective.Parameters; // current parameters + var Pstep = Vector.Build.Dense(P.Count); // the change of parameters + var RSS = objective.Residue; // Residual Sum of Squares = R'R + + if (maximumIterations < 0) + { + maximumIterations = (objective.IsJacobianSupported) + ? 100 * (initialGuess.Count + 1) + : 200 * (initialGuess.Count + 1); + } + + // if RSS == NaN, stop + if (double.IsNaN(RSS)) + { + exitCondition = ExitCondition.InvalidValues; + return new ModelMinimizationResult(objective, -1, exitCondition); + } + + // When only function evaluation is needed, set maximumIterations to zero, + if (maximumIterations == 0) + { + exitCondition = ExitCondition.ManuallyStopped; + } + + // if RSS <= fTol, stop + if (RSS <= functionTolerance) + { + exitCondition = ExitCondition.Converged; // SmallRSS + } + + // Evaluate gradient and Hessian + objective.EvaluateJacobian(P); + var Gradient = objective.Gradient; + var Hessian = objective.Hessian; + var diagonalOfHessian = Hessian.Diagonal(); // diag(H) + + // if ||g||oo <= gtol, found and stop + if (Gradient.InfinityNorm() <= gradientTolerance) + { + exitCondition = ExitCondition.RelativeGradient; + } + + if (exitCondition != ExitCondition.None) + { + objective.EvaluateCovariance(P); + return new ModelMinimizationResult(objective, -1, exitCondition); + } + + double mu = initialMu * diagonalOfHessian.Max(); // μ + double nu = 2; // ν + int iterations = 0; + while (iterations < maximumIterations && exitCondition == ExitCondition.None) + { + iterations++; + + while (true) + { + Hessian.SetDiagonal(Hessian.Diagonal() + mu); // hessian[i, i] = hessian[i, i] + mu; + + // solve normal equations + Pstep = Hessian.Solve(Gradient); + + // if ||ΔP|| <= xTol * (||P|| + xTol), found and stop + if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.DotProduct(P))) + { + exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + break; + } + + var Pnew = P + Pstep; // new parameters to test + + objective.EvaluateFunction(Pnew); + var RSSnew = objective.Residue; + + if (double.IsNaN(RSSnew)) + { + exitCondition = ExitCondition.InvalidValues; + break; + } + + // calculate the ratio of the actual to the predicted reduction. + // ρ = (RSS - RSSnew) / (Δp'(μΔp + g)) + var predictedReduction = Pstep.DotProduct(mu * Pstep + Gradient); + var rho = (predictedReduction != 0) + ? (RSS - RSSnew) / predictedReduction + : 0; + + if (rho > 0.0) + { + // accepted + Pnew.CopyTo(P); + RSS = RSSnew; + + // update gradient and Hessian + objective.EvaluateJacobian(P); + Gradient = objective.Gradient; + Hessian = objective.Hessian; + diagonalOfHessian = Hessian.Diagonal(); + + // if ||g||_oo <= gtol, found and stop + if (Gradient.InfinityNorm() <= gradientTolerance) + { + exitCondition = ExitCondition.RelativeGradient; + } + + // if ||R||^2 < fTol, found and stop + if (RSS <= functionTolerance) + { + exitCondition = ExitCondition.Converged; // SmallRSS + } + + mu = mu * Math.Max(1.0 / 3.0, 1.0 - Math.Pow(2.0 * rho - 1.0, 3)); + nu = 2; + + break; + } + else + { + // rejected, increased μ + mu = mu * nu; + nu = 2 * nu; + + Hessian.SetDiagonal(diagonalOfHessian); + } + } + } + + if (iterations >= maximumIterations) + { + exitCondition = ExitCondition.ExceedIterations; + } + + // finalize + objective.EvaluateCovariance(P); + + return new ModelMinimizationResult(objective, iterations, exitCondition); + } + } +} diff --git a/src/Numerics/Optimization/ModelMinimizationResult.cs b/src/Numerics/Optimization/ModelMinimizationResult.cs new file mode 100644 index 00000000..baaf3710 --- /dev/null +++ b/src/Numerics/Optimization/ModelMinimizationResult.cs @@ -0,0 +1,52 @@ +using MathNet.Numerics.LinearAlgebra; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace MathNet.Numerics.Optimization +{ + public class ModelMinimizationResult + { + public IObjectiveModel ModelInfoAtMinimum { get; private set; } + + /// + /// Returns the best fit parameters. + /// + public Vector BestFitParameters { get { return ModelInfoAtMinimum.Parameters; } } + + /// + /// Returns the standard errors of the corresponding parameters + /// + public Vector StandardErrors + { + get + { + if (ModelInfoAtMinimum.Covariance == null) + return null; + return ModelInfoAtMinimum.Covariance.Diagonal().PointwiseSqrt(); + } + } + + /// + /// Returns the y-values of the fitted model that correspond to the independent values. + /// + public Vector BestFitValues { get { return ModelInfoAtMinimum.Values; } } + + /// + /// Returns the residual sum of squares. + /// + public double Residue { get { return ModelInfoAtMinimum.Residue; } } + public double DegreeOfFreedom { get { return ModelInfoAtMinimum.DegreeOfFreedom; } } + + public int Iterations { get; private set; } + public ExitCondition ReasonForExit { get; private set; } + + public ModelMinimizationResult(IObjectiveModel modelInfo, int iterations, ExitCondition reasonForExit) + { + ModelInfoAtMinimum = modelInfo; + Iterations = iterations; + ReasonForExit = reasonForExit; + } + } +} diff --git a/src/Numerics/Optimization/ObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModel.cs new file mode 100644 index 00000000..631defae --- /dev/null +++ b/src/Numerics/Optimization/ObjectiveModel.cs @@ -0,0 +1,40 @@ +using MathNet.Numerics.LinearAlgebra; +using MathNet.Numerics.Optimization.ObjectiveModels; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace MathNet.Numerics.Optimization +{ + public static class ObjectiveModel + { + /// + /// Fitting model with a user supplied jacobian for non-linear least squares regression. + /// + public static IObjectiveModel FittingModel(Func, double, double> function, Func, double, Vector> derivatives, + Vector observedX, Vector observedY, Vector weight = null, + Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) + { + var objective = new FittingObjectiveModel(function, derivatives); + objective.SetObserved(observedX, observedY, weight); + objective.SetParameters(lowerBound, upperBound, scales, isFixed); + return objective; + } + + /// + /// Fitting model for non-linear least squares regression. + /// The numerical jacobian with accuracy order is used. + /// + public static IObjectiveModel FittingModel(Func, double, double> function, + Vector observedX, Vector observedY, Vector weight = null, + Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null, + int accuracyOrder = 2) + { + var objective = new FittingObjectiveModel(function, null, accuracyOrder: accuracyOrder); + objective.SetObserved(observedX, observedY, weight); + objective.SetParameters(lowerBound, upperBound, scales, isFixed); + return objective; + } + } +} diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs new file mode 100644 index 00000000..a1ea2beb --- /dev/null +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -0,0 +1,729 @@ +using MathNet.Numerics.LinearAlgebra; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace MathNet.Numerics.Optimization.ObjectiveModels +{ + internal class FittingObjectiveModel : IObjectiveModel + { + readonly Func, double, double> userFunction; // (p, x) => f(x; p) + readonly Func, double, Vector> userDerivatives; // (p, x) => df(x; p)/dp + + #region Public Variables + + /// + /// Set or get the values of the independent variable. + /// + public Vector ObservedX { get; private set; } + + /// + /// Set or get the values of the observations. + /// + public Vector ObservedY { get; private set; } + + /// + /// Set or get the values of the weights for the observations. + /// inverse of the standard measurement errors + /// If null, unity weighting is used. + /// + public Matrix Weights { get; private set; } + // W = LL' + private Vector L; + + /// + /// Set or get the values of the parameters. + /// + public Vector Parameters { get; private set; } + + /// + /// Set or get the values of the parameters. + /// + public List IsFixed { get; set; } + + /// + /// Set or get the values of the parameters. + /// + public Vector LowerBound { get; set; } + + /// + /// Set or get the values of the parameters. + /// + public Vector UpperBound { get; set; } + + /// + /// Set or get the scale factor of the parameters. + /// + public Vector Scales { get; set; } + + /// + /// Set of get whether or not the parameters are bounded. + /// + public bool IsBounded { get; set; } + + /// + /// Get the y-values of the fitted model that correspond to the independent values. + /// + public Vector Values { get; private set; } + + /// + /// Get the error values, R(x; p) = L * (y - f(x; p)) where L = sqrt(W) + /// + private Vector Residuals; + + /// + /// Get the residual sum of squares, R.DotProduct(R) + /// + public double Residue { get; private set; } + + /// + /// Get the Jacobian matrix of x and p, J(x; p). + /// + public Matrix Jacobian { get; private set; } + + /// + /// Get the Gradient vector of x and p, J'WR + /// + public Vector Gradient { get; private set; } + + /// + /// Get the Hessian matrix of x and p, J'WJ + /// + public Matrix Hessian { get; private set; } + + /// + /// Get the number of observations. + /// + public int NumberOfObservations { get { return (ObservedY == null) ? 0 : ObservedY.Count; } } + + /// + /// Get the number of unknown parameters. + /// + public int NumberOfParameters { get { return (Parameters == null) ? 0 : Parameters.Count; } } + + /// + /// Get the degree of freedom + /// + public int DegreeOfFreedom + { + get + { + var dof = NumberOfObservations - NumberOfParameters; + if (IsFixed != null) + { + dof = dof + IsFixed.Count(p => p == true); + } + return dof; + } + } + + /// + /// Get the covariance matrix. + /// + public Matrix Covariance { get; private set; } + + /// + /// Get the number of calls to function. + /// + public int FunctionEvaluations { get; private set; } + /// + /// Get the number of calls to jacobian. + /// + public int JacobianEvaluations { get; private set; } + + /// + /// Set or get the desired accuracy order of the numerical jacobian. + /// + public int AccuracyOrder { get; set; } + + /// + /// Get whether or not the analytical jacobian is supported. + /// + public bool IsJacobianSupported { get { return userDerivatives != null; } } + + #endregion Public Variables + + public FittingObjectiveModel(Func, double, double>function, Func, double, Vector> derivatives, int accuracyOrder = 2) + { + userFunction = function; + userDerivatives = derivatives; + AccuracyOrder = Math.Min(6, Math.Max(1, accuracyOrder)); + } + + public IObjectiveModel Fork() + { + return new FittingObjectiveModel(userFunction, userDerivatives, AccuracyOrder) + { + ObservedX = ObservedX, + ObservedY = ObservedY, + Weights = Weights, + + Parameters = Parameters, + LowerBound = LowerBound, + UpperBound = UpperBound, + IsFixed = IsFixed, + Scales = Scales, + IsBounded = IsBounded, + + Residue = Residue, + Jacobian = Jacobian + }; + } + + public IObjectiveModel CreateNew() + { + return new FittingObjectiveModel(userFunction, userDerivatives); + } + + /// + /// Set observed data to fit. + /// + public void SetObserved(Vector observedX, Vector observedY, Vector weights = null) + { + if (observedX == null || observedY == null) + { + throw new ArgumentNullException("The data set can't be null."); + } + if (observedX.Count != observedY.Count) + { + throw new ArgumentException("The observed x data can't have different from observed y data."); + } + ObservedX = observedX; + ObservedY = observedY; + + if (weights != null && weights.Count != observedY.Count) + { + throw new ArgumentException("The weightings can't have different from observations."); + } + if (weights != null && weights.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) + { + throw new ArgumentException("The weightings are not well-defined."); + } + if (weights != null && weights.Count(x => x == 0) == weights.Count) + { + throw new ArgumentException("All the weightings can't be zero."); + } + if (weights != null && weights.Count(x => x < 0) > 0) + { + weights = weights.PointwiseAbs(); + } + + Weights = (weights == null) + ? null + : Matrix.Build.DenseOfDiagonalVector(weights); + + L = (weights == null) + ? null + : Weights.Diagonal().PointwiseSqrt(); + } + + /// + /// Set observed data to fit. + /// + public void SetObserved(double[] observedX, double[] observedY, double[] weights = null) + { + if (observedX == null || observedY == null) + { + throw new ArgumentNullException("The data set can't be null."); + } + if (observedX.Length != observedY.Length) + { + throw new ArgumentException("The observed x data can't have different from observed y data."); + } + + var wVector = (weights == null) + ? null + : Vector.Build.DenseOfArray(weights); + SetObserved(Vector.Build.DenseOfArray(observedX), Vector.Build.DenseOfArray(observedY), wVector); + } + + /// + /// Set parameters. + /// + /// If bounded, the paramneters will be projected to unconstrained range by the mapping rule from the MINPACK. + /// If the projection is not needed, set IsBounded = false befre calling the Minimization method. + /// + /// The lower bounds of parameters. + /// The upper bounds of parameters. + /// /// The scaling constants of parameters + /// The list to the parameters fix or free. + public void SetParameters(Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) + { + if (lowerBound != null && lowerBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) + { + throw new ArgumentException("The lower bounds must be finite."); + } + LowerBound = lowerBound; + + if (upperBound != null && upperBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) + { + throw new ArgumentException("The upper bounds must be finite."); + } + if (upperBound != null && lowerBound != null && upperBound.Count != lowerBound.Count) + { + throw new ArgumentException("The upper bounds can't have different elements from the lower bounds."); + } + UpperBound = upperBound; + + if (scales != null && scales.Count(x => double.IsInfinity(x) || double.IsNaN(x) || x == 0) > 0) + { + throw new ArgumentException("The scales must be finite."); + } + if (scales != null && lowerBound != null && scales.Count != lowerBound.Count) + { + throw new ArgumentException("The upper bounds can't have different elements from the lower bounds."); + } + if (scales != null && upperBound != null && scales.Count != upperBound.Count) + { + throw new ArgumentException("The upper bounds can't have different elements from the upper bounds."); + } + if (scales != null && scales.Count(x => x < 0) > 0) + { + scales.PointwiseAbs(); + } + Scales = scales; + + IsBounded = (LowerBound != null || UpperBound != null || Scales != null); + + if (isFixed != null && lowerBound != null && isFixed.Count != lowerBound.Count) + { + throw new ArgumentException("The initial guess can't have different elements from the lower bounds."); + } + if (isFixed != null && upperBound != null && isFixed.Count != upperBound.Count) + { + throw new ArgumentException("The initial guess can't have different elements from the upper bounds."); + } + if (isFixed != null && scales != null && isFixed.Count != scales.Count) + { + throw new ArgumentException("The initial guess can't have different elements from the scales."); + } + if (isFixed != null && isFixed.Count(p => p == true) == isFixed.Count) + { + throw new ArgumentException("All the parameters can't be fixed."); + } + IsFixed = isFixed; + } + + /// + /// Set parameters. + /// + /// If bounded, the paramneters will be projected to unconstrained range by the mapping rule. + /// If the projection is not needed, set IsBounded = false befre calling the Minimization method. + /// + /// The lower bounds of parameters. + /// The upper bounds of parameters. + /// The scaling constants of parameters + /// The list to the parameters fix or free. + public void SetParameters(double[] lowerBound = null, double[] upperBound = null, double[] scales = null, bool[] isFixed = null) + { + var lb = (lowerBound == null) ? null : Vector.Build.DenseOfArray(lowerBound); + var ub = (upperBound == null) ? null : Vector.Build.DenseOfArray(upperBound); + var sc = (scales == null) ? null : Vector.Build.DenseOfArray(scales); + var fp = (isFixed == null) ? null : isFixed.ToList(); + + SetParameters(lb, ub, sc, fp); + } + + public void EvaluateFunction(Vector parameters) + { + ValidateParameters(parameters); + + // To handle the box constrained minimization as the unconstrained minimization, + // parameters are mapping by the following rule. + // + // 1. lower < P < upper + // Pint = asin(2 * (Pext - lower) / (upper - lower) - 1) + // Pext = lower + (sin(Pint) + 1) * (upper - lower) / 2 + // dPext/dPint = (upper - lower) / 2 * cos(Pint) + // 2. lower < P + // Pint = sqrt((Pext - lower + 1)^2 - 1) + // Pext = lower - 1 + sqrt(Pint^2 + 1) + // dPext/dPint = Pint / sqrt(Pint^2 + 1) + // 3. P < upper + // Pint = sqrt((upper - Pext + 1)^2 - 1) + // Pext = upper + 1 - sqrt(Pint^2 + 1) + // dPext/dPint = - Pint / sqrt(Pint^2 + 1) + // 4. no bounds, but scales + // Pint = Pext / scale + // Pext = Pint * scale + // dPext/dPint = scale + // + // see ProjectParametersToInternal(Pext), ProjectParametersToExternal(Pint), ScaleFactorsOfJacobian(Pint) methods. + // + // References: + // [1] https://lmfit.github.io/lmfit-py/bounds.html + // + // + // Except when it is initial guess, the parameters argument is always internal parameter. + // So, first map the parameters argument to the external parameters in order to calculate function values. + var Pext = (FunctionEvaluations > 0 && this.IsBounded) + ? ProjectParametersToExternal(parameters) + : parameters.Clone(); + + // Project parameters, now this.Parameters are the internal parameters. + Parameters = (this.IsBounded) + ? ProjectParametersToInternal(Pext) + : Pext; + + // Calculates the residuals, (y[i] - f(x[i]; p)) * L[i] + if (Values == null) + { + Values = Vector.Build.Dense(NumberOfObservations); + } + for (int i = 0; i < NumberOfObservations; i++) + { + Values[i] = userFunction(Pext, ObservedX[i]); + } + FunctionEvaluations++; + + // calculate the weighted residuals + Residuals = (Weights == null) + ? ObservedY - Values + : (ObservedY - Values).PointwiseMultiply(L); + + // Calculate the residual sum of squares + Residue = Residuals.DotProduct(Residuals); + + return; + } + + public void EvaluateJacobian(Vector parameters) + { + var Pext = (IsBounded) + ? ProjectParametersToExternal(parameters) + : parameters.Clone(); + + // Calculates the jacobian of x and p. + if (userDerivatives != null) + { + // analytical jacobian + if (Jacobian == null) + { + Jacobian = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); + } + for (int i = 0; i < NumberOfObservations; i++) + { + Jacobian.SetRow(i, userDerivatives(Pext, ObservedX[i])); + } + JacobianEvaluations++; + } + else + { + // numerical jacobian + Jacobian = NumericalJacobian(Pext, Values, AccuracyOrder); + FunctionEvaluations += AccuracyOrder; + } + + var scaleFactors = (this.IsBounded) + ? ScaleFactorsOfJacobian(Parameters) + : Vector.Build.Dense(Parameters.Count, 1.0); + + // Jint(x; Pint) = Jext(x; Pext) * scale where scale = dPext/dPint + for (int i = 0; i < NumberOfObservations; i++) + { + for (int j = 0; j < NumberOfParameters; j++) + { + if (IsFixed != null && IsFixed[j]) + { + // if j-th parameter is fixed, set J[i, j] = 0 + Jacobian[i, j] = 0.0; + } + else + { + Jacobian[i, j] = Jacobian[i, j] * scaleFactors[j]; + } + } + } + + // Gradient, g = J'W(y − f(x; p)) = J'L(L'E) = J'LR + Gradient = (Weights == null) + ? Jacobian.Transpose() * (ObservedY - Values) + : Jacobian.Transpose() * Weights * (ObservedY - Values); + + // approximated Hessian, H = J'WJ + ∑LRiHi ~ J'WJ near the minimum + Hessian = (Weights == null) + ? Jacobian.Transpose() * Jacobian + : Jacobian.Transpose() * Weights * Jacobian; + } + + public void EvaluateCovariance(Vector parameters) + { + // convert to bounded(external) parameters + var Pext = (IsBounded) + ? ProjectParametersToExternal(parameters) + : parameters.Clone(); + + // set IsBounded = false to get external Parameters and covariance matrix + this.IsBounded = false; + + EvaluateFunction(Pext); + EvaluateJacobian(Pext); + + if (Hessian == null || Residuals == null || DegreeOfFreedom < 1) + { + Covariance = null; + return; + } + + var covariance = Hessian.PseudoInverse() * Residuals.DotProduct(Residuals) / DegreeOfFreedom; + + Covariance = covariance; + + // restore isBounded + this.IsBounded = (LowerBound != null || UpperBound != null); + + return; + } + + private void ValidateParameters(Vector parameters) + { + if (parameters == null) + { + throw new ArgumentNullException("parameters"); + } + else if (parameters.Count(p => double.IsNaN(p) || double.IsInfinity(p)) > 0) + { + throw new ArgumentException("the parameters must be finite."); + } + if (LowerBound != null && parameters.Count != LowerBound.Count) + { + throw new ArgumentException("The parameters can't have different size from the lower bounds."); + } + if (UpperBound != null && parameters.Count != UpperBound.Count) + { + throw new ArgumentException("The parameters can't have different size from the upper bounds."); + } + if (Scales != null && parameters.Count != Scales.Count) + { + throw new ArgumentException("The parameters can't have different size from the scales."); + } + if (IsFixed != null && parameters.Count != IsFixed.Count) + { + throw new ArgumentException("The parameters can't have different size from the IsFixed list."); + } + } + + #region Numerical Derivatives + + // Numerical derivatives by using the central or forward finite difference + private Matrix NumericalJacobian(Vector parameters, Vector currentValues, int accuracyOrder = 2) + { + const double sqrtEpsilon = 1.4901161193847656250E-8; // sqrt(machineEpsilon) + + Matrix derivertives = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); + + var d = 0.000003 * parameters.PointwiseAbs().PointwiseMaximum(sqrtEpsilon); + + var h = Vector.Build.Dense(NumberOfParameters); + for (int i = 0; i < NumberOfObservations; i++) + { + var x = ObservedX[i]; + for (int j = 0; j < NumberOfParameters; j++) + { + h[j] = d[j]; + + if (accuracyOrder >= 6) + { + // f'(x) = {- f(x - 3h) + 9f(x - 2h) - 45f(x - h) + 45f(x + h) - 9f(x + 2h) + f(x + 3h)} / 60h + O(h^6) + var f1 = userFunction(parameters - 3 * h, x); + var f2 = userFunction(parameters - 2 * h, x); + var f3 = userFunction(parameters - h, x); + var f4 = userFunction(parameters + h, x); + var f5 = userFunction(parameters + 2 * h, x); + var f6 = userFunction(parameters + 3 * h, x); + + var prime = (-f1 + 9 * f2 - 45 * f3 + 45 * f4 - 9 * f5 + f6) / (60 * h[j]); + derivertives[i, j] = prime; + } + else if (accuracyOrder == 5) + { + // f'(x) = {-137f(x) + 300f(x + h) - 300f(x + 2h) + 200f(x + 3h) - 75f(x + 4h) + 12f(x + 5h)} / 60h + O(h^5) + var f1 = currentValues[i]; + var f2 = userFunction(parameters + h, x); + var f3 = userFunction(parameters + 2 * h, x); + var f4 = userFunction(parameters + 3 * h, x); + var f5 = userFunction(parameters + 4 * h, x); + var f6 = userFunction(parameters + 5 * h, x); + + var prime = (-137 * f1 + 300 * f2 - 300 * f3 + 200 * f4 - 75 * f5 + 12 * f6) / (60 * h[j]); + derivertives[i, j] = prime; + } + else if (accuracyOrder == 4) + { + // f'(x) = {f(x - 2h) - 8f(x - h) + 8f(x + h) - f(x + 2h)} / 12h + O(h^4) + var f1 = userFunction(parameters - 2 * h, x); + var f2 = userFunction(parameters - h, x); + var f3 = userFunction(parameters + h, x); + var f4 = userFunction(parameters + 2 * h, x); + + var prime = (f1 - 8 * f2 + 8 * f3 - f4) / (12 * h[j]); + derivertives[i, j] = prime; + } + else if (accuracyOrder == 3) + { + // f'(x) = {-11f(x) + 18f(x + h) - 9f(x + 2h) + 2f(x + 3h)} / 6h + O(h^3) + var f1 = currentValues[i]; + var f2 = userFunction(parameters + h, x); + var f3 = userFunction(parameters + 2 * h, x); + var f4 = userFunction(parameters + 3 * h, x); + + var prime = (-11 * f1 + 18 * f2 - 9 * f3 + 2 * f4) / (6 * h[j]); + derivertives[i, j] = prime; + } + else if (accuracyOrder == 2) + { + // f'(x) = {f(x + h) - f(x - h)} / 2h + O(h^2) + var f1 = userFunction(parameters + h, x); + var f2 = userFunction(parameters - h, x); + + var prime = (f1 - f2) / (2 * h[j]); + derivertives[i, j] = prime; + } + else + { + // f'(x) = {- f(x) + f(x + h)} / h + O(h) + var f1 = currentValues[i]; + var f2 = userFunction(parameters + h, x); + + var prime = (-f1 + f2) / h[j]; + derivertives[i, j] = prime; + } + + h[j] = 0; + } + } + + return derivertives; + } + + #endregion Numerical Derivatives + + #region Projection + + private Vector ProjectParametersToInternal(Vector Pext) + { + var Pint = Pext.Clone(); + + if (LowerBound != null && UpperBound != null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = Math.Asin((2.0 * (Pext[i] - LowerBound[i]) / (UpperBound[i] - LowerBound[i])) - 1.0); + } + + return Pint; + } + else if (LowerBound != null && UpperBound == null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = Math.Sqrt(Math.Pow(Pext[i] - LowerBound[i] + 1.0, 2) - 1.0); + } + + return Pint; + } + else if (LowerBound == null && UpperBound != null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = Math.Sqrt(Math.Pow(UpperBound[i] - Pext[i] + 1.0, 2) - 1.0); + } + + return Pint; + } + else if (Scales != null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = Pext[i] / Scales[i]; + } + + return Pint; + } + + return Pint; + } + + private Vector ProjectParametersToExternal(Vector Pint) + { + var Pext = Pint.Clone(); + + if (LowerBound != null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = LowerBound[i] + (UpperBound[i] / 2.0 - LowerBound[i] / 2.0) * (Math.Sin(Pint[i]) + 1.0); + } + + return Pext; + } + else if (LowerBound != null && UpperBound == null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = LowerBound[i] + Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0; + } + + return Pext; + } + else if (LowerBound == null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = UpperBound[i] - Math.Sqrt(Pint[i] * Pint[i] + 1.0) + 1.0; + } + + return Pext; + } + else if (Scales != null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = Pint[i] * Scales[i]; + } + + return Pext; + } + + return Pext; + } + + private Vector ScaleFactorsOfJacobian(Vector Pint) + { + var scale = Vector.Build.Dense(Pint.Count, 1.0); + + if (LowerBound != null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + scale[i] = (UpperBound[i] - LowerBound[i]) / 2.0 * Math.Cos(Pint[i]); + } + return scale; + } + else if (LowerBound != null && UpperBound == null) + { + for (int i = 0; i < Pint.Count; i++) + { + scale[i] = Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); + } + return scale; + } + else if (LowerBound == null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + scale[i] = -Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); + } + return scale; + } + else if (Scales != null) + { + return Scales; + } + + return scale; + } + + #endregion Projection + } +} diff --git a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs new file mode 100644 index 00000000..a8ab570f --- /dev/null +++ b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs @@ -0,0 +1,322 @@ +using MathNet.Numerics.LinearAlgebra; +using System; + +namespace MathNet.Numerics.Optimization +{ + public sealed class TrustRegionDogLegMinimizer + { + #region Tolerances and options + + /// + /// The stopping threshold for infinity norm of the gradient. + /// + public static double GradientTolerance { get; set; } + + /// + /// The stopping threshold for L2 norm of the change of the parameters. + /// + public static double StepTolerance { get; set; } + + /// + /// The stopping threshold for the function value or L2 norm of the residuals. + /// + public static double FunctionTolerance { get; set; } + + /// + /// The stopping threshold for the trust region radius. + /// + public static double RadiusTolerance { get; set; } + + /// + /// The maximum number of iterations. + /// + public int MaximumIterations { get; set; } + + #endregion Tolerances and options + + public TrustRegionDogLegMinimizer(double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) + { + FunctionTolerance = functionTolerance; + GradientTolerance = gradientTolerance; + StepTolerance = stepTolerance; + RadiusTolerance = radiusTolerance; + MaximumIterations = maximumIterations; + } + + public ModelMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + { + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + return Minimum(objective, initialGuess, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); + } + + public ModelMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + { + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + return Minimum(objective, CreateVector.DenseOfArray(initialGuess), GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); + } + + /// + /// Non-linear least square fitting by the trust-region dogleg algorithm. + /// + /// The objective model, including function, jacobian, observations, and parameter bounds. + /// The initial guess values. + /// The stopping threshold for L2 norm of the residuals. + /// The stopping threshold for infinity norm of the gradient vector. + /// The stopping threshold for L2 norm of the change of parameters. + /// The stopping threshold for trust region radius + /// The max iterations. + /// + public static ModelMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) + { + // Non-linear least square fitting by the trust-region dogleg algorithm. + // + // The Powell's dogleg method is finding the minimum of a function F(p) that is a sum of squares of nonlinear functions. + // + // For given datum pair (x, y), uncertainties σ (or weighting W = 1 / σ^2) and model function f = f(x; p), + // let's find the parameters of the model so that the sum of the quares of the deviations is minimized. + // + // F(p) = 1/2 * ∑{ Wi * (yi - f(xi; p))^2 } + // pbest = argmin F(p) + // + // Here, we will use the following terms: + // Weighting W is the diagonal matrix and can be decomposed as LL', so L = 1/σ + // Residuals, R = L(y - f(x; p)) + // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) + // Jacobian J = df(x; p)/dp + // Gradient g = J'W(y − f(x; p)) = J'LR + // Approximated Hessian H = J'WJ + // + // The Powell's dogleg algorithm is summarized as follows: + // initially set trust-region radius, Δ + // repeat + // solve quadratic subproblem + // update Δ: + // let ρ = (RSS - RSSnew) / predRed + // if ρ > 0.75, Δ = 2Δ + // if ρ < 0.25, Δ = Δ/4 + // if ρ > eta, P = P + ΔP + // + // References: + // [1]. Madsen, K., H. B. Nielsen, and O. Tingleff. + // "Methods for Non-Linear Least Squares Problems. Technical University of Denmark, 2004. Lecture notes." (2004). + // Available Online from: http://orbit.dtu.dk/files/2721358/imm3215.pdf + // [2]. SciPy + // Available Online from: https://github.com/scipy/scipy/blob/master/scipy/optimize/_trustregion_dogleg.py + + double maxDelta = 1000; + double eta = 0; + + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + ExitCondition exitCondition = ExitCondition.None; + + // First, calculate function values and setup variables + objective.EvaluateFunction(initialGuess); + var P = objective.Parameters; // current parameters + var RSS = objective.Residue; // Residual Sum of Squares = R'R + var RSSinit = RSS; // RSS at initial gussing parameters + + if (maximumIterations < 0) + { + maximumIterations = 200 * (initialGuess.Count + 1); + } + + // if R == NaN, stop + if (double.IsNaN(RSS)) + { + exitCondition = ExitCondition.InvalidValues; + return new ModelMinimizationResult(objective, -1, exitCondition); + } + + // When only function evaluation is needed, set maximumIterations to zero, + if (maximumIterations == 0) + { + exitCondition = ExitCondition.ManuallyStopped; + } + + // if ||R||^2 <= fTol, stop + if (RSS <= functionTolerance) + { + exitCondition = ExitCondition.Converged; // SmallRSS + } + + // Evaluate projected Hessian, and gradient + objective.EvaluateJacobian(P); + var Hessian = objective.Hessian; + var Gradient = objective.Gradient; + + // if ||g||_oo <= gtol, found and stop + if (Gradient.InfinityNorm() <= gradientTolerance) + { + exitCondition = ExitCondition.RelativeGradient; // SmallGradient + } + + if (exitCondition != ExitCondition.None) + { + // finalize + objective.EvaluateCovariance(P); + return new ModelMinimizationResult(objective, -1, exitCondition); + } + + // initialize trust-region radius, Δ + double delta = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); + delta = Math.Max(1.0, Math.Min(delta, maxDelta)); + + int iterations = 0; + while (iterations < maximumIterations && exitCondition == ExitCondition.None) + { + iterations++; + + // solve the subproblem + var subprogram = SolveQuadraticSubproblem(objective, delta); + var Pstep = subprogram.Item1; + var predictedReduction = subprogram.Item2; + var hitBoundary = subprogram.Item3; + + if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) + { + exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + break; + } + + var Pnew = P + Pstep; // parameters to test + + objective.EvaluateFunction(Pnew); + var RSSnew = objective.Residue; + + // calculate the ratio of the actual to the predicted reduction. + double rho = (predictedReduction != 0) + ? (RSS - RSSnew) / predictedReduction + : 0; + + if (rho > 0.75 && hitBoundary) + { + delta = Math.Min(2.0 * delta, maxDelta); + } + else if (rho < 0.25) + { + delta = delta * 0.25; + if (delta <= radiusTolerance * (radiusTolerance + P.DotProduct(P))) + { + exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + break; + } + } + + if (rho > eta) + { + // accepted + Pnew.CopyTo(P); + RSS = RSSnew; + + // update Jacobian, Hessian, and gradient + objective.EvaluateJacobian(P); + Gradient = objective.Gradient; + Hessian = objective.Hessian; + + // if ||g||_oo <= gtol, found and stop + if (Gradient.InfinityNorm() <= gradientTolerance) + { + exitCondition = ExitCondition.RelativeGradient; + } + + // if ||R||^2 < fTol, found and stop + if (RSS <= functionTolerance) + { + exitCondition = ExitCondition.Converged; // SmallRSS + } + } + } + + if (iterations >= maximumIterations) + { + exitCondition = ExitCondition.ExceedIterations; + } + + // finalize + objective.EvaluateCovariance(P); + + return new ModelMinimizationResult(objective, iterations, exitCondition); + } + + private static Tuple, double, bool> SolveQuadraticSubproblem(IObjectiveModel objective, double delta) + { + Vector Pstep; + double predictedReduction; + bool hitBoundary = false; + + var Jacobian = objective.Jacobian; + var Gradient = objective.Gradient; + var Hessian = objective.Hessian; + var RSS = objective.Residue; + + // newton point + // the Gauss–Newton step by solving the normal equations + var Pgn = Hessian.PseudoInverse() * Gradient; // Hessian.Solve(Gradient) fails so many times... + + // cauchy point + // steepest descent direction is given by + var alpha = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); + var Psd = alpha * Gradient; + + // update step and prectted reduction + if (Pgn.L2Norm() <= delta) + { + // Pgn is inside trust region radius + hitBoundary = false; + Pstep = Pgn; + predictedReduction = RSS; + } + else if (alpha * Psd.L2Norm() >= delta) + { + // Psd is outside trust region radius + hitBoundary = true; + Pstep = delta / Psd.L2Norm() * Psd; + predictedReduction = delta * (2.0 * (alpha * Gradient).L2Norm() - delta) / 2.0 / alpha; + } + else + { + // Pstep is intersection of the trust region boundary + hitBoundary = true; + var beta = FindBeta(alpha, Psd, Pgn, delta); + Pstep = alpha * Psd + beta * (Pgn - alpha * Psd); + predictedReduction = 0.5 * alpha * (1 - beta) * (1 - beta) * Gradient.DotProduct(Gradient) + beta * (2 - beta) * RSS; + } + + return new Tuple, double, bool>(Pstep, predictedReduction, hitBoundary); + } + + private static double FindBeta(double alpha, Vector sd, Vector gn, double delta) + { + // Pstep is intersection of the trust region boundary + // Pstep = α*Psd + β*(Pgn - α*Psd) + // find r so that ||Pstep|| = Δ + // z = α*Psd, d = (Pgn - z) + // (d^2)β^2 + (2*z*d)β + (z^2 - Δ^2) = 0 + // get positive β by using the quadratic formula + + var z = alpha * sd; + var d = gn - z; + + var a = d.DotProduct(d); + var b = 2.0 * z.DotProduct(d); + var c = z.DotProduct(z) - delta * delta; + + var aux = b + ((b >= 0) ? 1.0 : -1.0) * Math.Sqrt(b * b - 4.0 * a * c); + var beta = Math.Max(-aux / 2.0 / a, -2.0 * c / aux); + + return beta; + } + } +} From 220ab5aadb1550cedb1b50aadef70fe4b3cccd0f Mon Sep 17 00:00:00 2001 From: diluculo Date: Mon, 31 Dec 2018 17:04:41 +0900 Subject: [PATCH 02/13] Updated the box-constrained mapping rule in the FittingObjectiveModel. --- .../NonLinearCurveFittingTests.cs | 178 ++++++++++++------ .../ObjectiveModels/FittingObjectiveModel.cs | 52 +++-- 2 files changed, 155 insertions(+), 75 deletions(-) diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs index fbbd3448..4918a263 100644 --- a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -42,11 +42,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + } } [Test] @@ -54,11 +55,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder:2); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + } } [Test] @@ -67,11 +69,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY, lowerBound : RosebbrockLowerBound, upperBound : RosenbrockUpperBound); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + } } [Test] @@ -79,13 +82,14 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound, - accuracyOrder: 2); + accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 3); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 3); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + } } [Test] @@ -93,11 +97,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 1); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); + } } [Test] @@ -105,11 +110,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[0], result.BestFitParameters[0], 1); - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[1], result.BestFitParameters[1], 1); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); + } } // model: Rat43 (https://www.itl.nist.gov/div898/strd/nls/data/ratkowsky3.shtml) @@ -147,7 +153,6 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, Rat43Start1); for (int i = 0; i < result.BestFitParameters.Count; i++) @@ -162,7 +167,6 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); - var result = solver.FindMinimum(obj, Rat43Start2); for (int i = 0; i < result.BestFitParameters.Count; i++) @@ -199,23 +203,22 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector BoxBodStart1 = new DenseVector(new double[] { 1.0, 1.0 }); private Vector BoxBodStart2 = new DenseVector(new double[] { 100.0, 0.75 }); - private Vector BoxBodLowerBound = new DenseVector(new double[] { 0, 0 }); - private Vector BoxBodUpperBound = new DenseVector(new double[] { 500.0, 10 }); - private Vector BoxBodScales = new DenseVector(new double[] { 100.0, 1 }); + private Vector BoxBodLowerBound = new DenseVector(new double[] { -1000, -100 }); + private Vector BoxBodUpperBound = new DenseVector(new double[] { 1000.0, 100 }); + private Vector BoxBodScales = new DenseVector(new double[] { 100.0, 0.1 }); [Test] public void LMDER_FindMinimum_BoxBod_Unconstrained() { var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); - - AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } } [Test] @@ -223,30 +226,96 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder:6); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); - - AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } } [Test] public void LMDER_FindMinimum_BoxBod_BoxConstrained() { + // lower < parameters < upper + // Note that in this case, scales have no effect. + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } - AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + // lower < parameters, no scales + + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + lowerBound: BoxBodLowerBound); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } + + // lower < parameters, scales + + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + lowerBound: BoxBodLowerBound, scales: BoxBodScales); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } + + // parameters < upper, no scales + + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + upperBound: BoxBodUpperBound); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } + + // parameters < upper, scales + + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + upperBound: BoxBodUpperBound, scales: BoxBodScales); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } + + // only scales + + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + scales: BoxBodScales); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } } [Test] @@ -256,14 +325,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 6); - - AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 6); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } } [Test] @@ -271,14 +339,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[0], result.BestFitParameters[0], 3); - AssertHelpers.AlmostEqualRelative(BoxBodPbest[1], result.BestFitParameters[1], 3); - - AssertHelpers.AlmostEqualRelative(BoxBodPstd[0], result.StandardErrors[0], 3); - AssertHelpers.AlmostEqualRelative(BoxBodPstd[1], result.StandardErrors[1], 3); + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 3); + } } // model : Thurber (https://www.itl.nist.gov/div898/strd/nls/data/thurber.shtml) @@ -359,7 +426,6 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberPrime, ThurberX, ThurberY); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); for (int i = 0; i < result.BestFitParameters.Count; i++) @@ -374,7 +440,6 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); for (int i = 0; i < result.BestFitParameters.Count; i++) @@ -391,7 +456,6 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests scales: ThurberScales, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); for (int i = 0; i < result.BestFitParameters.Count; i++) diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs index a1ea2beb..ce81fb54 100644 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -329,30 +329,34 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels ValidateParameters(parameters); // To handle the box constrained minimization as the unconstrained minimization, - // parameters are mapping by the following rule. + // the parameters are mapping by the following rules, + // which are modified the rules shown in the ref[1] in order to introduce scales. // - // 1. lower < P < upper + // 1. lower < Pext < upper // Pint = asin(2 * (Pext - lower) / (upper - lower) - 1) // Pext = lower + (sin(Pint) + 1) * (upper - lower) / 2 // dPext/dPint = (upper - lower) / 2 * cos(Pint) - // 2. lower < P - // Pint = sqrt((Pext - lower + 1)^2 - 1) - // Pext = lower - 1 + sqrt(Pint^2 + 1) - // dPext/dPint = Pint / sqrt(Pint^2 + 1) - // 3. P < upper - // Pint = sqrt((upper - Pext + 1)^2 - 1) - // Pext = upper + 1 - sqrt(Pint^2 + 1) - // dPext/dPint = - Pint / sqrt(Pint^2 + 1) + // + // 2. lower < Pext + // Pint = sqrt((Pext/scale - lower/scale + 1)^2 - 1) + // Pext = lower + scale * (sqrt(Pint^2 + 1) - 1) + // dPext/dPint = scale * Pint / sqrt(Pint^2 + 1) + // + // 3. Pext < upper + // Pint = sqrt((upper / scale - Pext / scale + 1)^2 - 1) + // Pext = upper + scale - scale * sqrt(Pint^2 + 1) + // dPext/dPint = - scale * Pint / sqrt(Pint^2 + 1) + // // 4. no bounds, but scales // Pint = Pext / scale // Pext = Pint * scale // dPext/dPint = scale // - // see ProjectParametersToInternal(Pext), ProjectParametersToExternal(Pint), ScaleFactorsOfJacobian(Pint) methods. + // The rules are applied in ProjectParametersToInternal, ProjectParametersToExternal, and ScaleFactorsOfJacobian methods. // // References: // [1] https://lmfit.github.io/lmfit-py/bounds.html - // + // [2] MINUIT User's Guide, https://root.cern.ch/download/minuit.pdf // // Except when it is initial guess, the parameters argument is always internal parameter. // So, first map the parameters argument to the external parameters in order to calculate function values. @@ -617,7 +621,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { for (int i = 0; i < Pext.Count; i++) { - Pint[i] = Math.Sqrt(Math.Pow(Pext[i] - LowerBound[i] + 1.0, 2) - 1.0); + Pint[i] = (Scales == null) + ? Math.Sqrt(Math.Pow(Pext[i] - LowerBound[i] + 1.0, 2) - 1.0) + : Math.Sqrt(Math.Pow((Pext[i] - LowerBound[i]) / Scales[i] + 1.0, 2) - 1.0); } return Pint; @@ -626,7 +632,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { for (int i = 0; i < Pext.Count; i++) { - Pint[i] = Math.Sqrt(Math.Pow(UpperBound[i] - Pext[i] + 1.0, 2) - 1.0); + Pint[i] = (Scales == null) + ? Math.Sqrt(Math.Pow(UpperBound[i] - Pext[i] + 1.0, 2) - 1.0) + : Math.Sqrt(Math.Pow((UpperBound[i] - Pext[i]) / Scales[i] + 1.0, 2) - 1.0); } return Pint; @@ -661,7 +669,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { for (int i = 0; i < Pint.Count; i++) { - Pext[i] = LowerBound[i] + Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0; + Pext[i] = (Scales == null) + ? LowerBound[i] + Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0 + : LowerBound[i] + Scales[i] * (Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0); } return Pext; @@ -670,7 +680,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { for (int i = 0; i < Pint.Count; i++) { - Pext[i] = UpperBound[i] - Math.Sqrt(Pint[i] * Pint[i] + 1.0) + 1.0; + Pext[i] = (Scales == null) + ? UpperBound[i] - Math.Sqrt(Pint[i] * Pint[i] + 1.0) + 1.0 + : UpperBound[i] - Scales[i] * (Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0); } return Pext; @@ -704,7 +716,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { for (int i = 0; i < Pint.Count; i++) { - scale[i] = Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); + scale[i] = (Scales == null) + ? Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0) + : Scales[i] * Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); } return scale; } @@ -712,7 +726,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { for (int i = 0; i < Pint.Count; i++) { - scale[i] = -Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); + scale[i] = (Scales == null) + ? -Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0) + : -Scales[i] * Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); } return scale; } From 4bac95e481e8bc22ff62953bca2eeb7e18a05188 Mon Sep 17 00:00:00 2001 From: diluculo Date: Mon, 31 Dec 2018 20:01:37 +0900 Subject: [PATCH 03/13] Added initialization step to ensure that the initialGuess is not internal parameters. --- src/Numerics/Optimization/IObjectiveModel.cs | 4 ++-- src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs | 4 ++++ .../Optimization/ObjectiveModels/FittingObjectiveModel.cs | 4 ++-- src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Numerics/Optimization/IObjectiveModel.cs b/src/Numerics/Optimization/IObjectiveModel.cs index 45f94409..72c15391 100644 --- a/src/Numerics/Optimization/IObjectiveModel.cs +++ b/src/Numerics/Optimization/IObjectiveModel.cs @@ -41,11 +41,11 @@ namespace MathNet.Numerics.Optimization /// /// Get the number of calls to function. /// - int FunctionEvaluations { get; } + int FunctionEvaluations { get; set; } /// /// Get the number of calls to jacobian. /// - int JacobianEvaluations { get; } + int JacobianEvaluations { get; set; } /// /// Get the degree of freedom. diff --git a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs index 681eb3dd..2b54d194 100644 --- a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs +++ b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs @@ -120,6 +120,10 @@ namespace MathNet.Numerics.Optimization ExitCondition exitCondition = ExitCondition.None; + // Initialize objective + objective.FunctionEvaluations = 0; + objective.JacobianEvaluations = 0; + // First, calculate function values and setup variables objective.EvaluateFunction(initialGuess); var P = objective.Parameters; // current parameters diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs index ce81fb54..6b9fba7c 100644 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -125,11 +125,11 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels /// /// Get the number of calls to function. /// - public int FunctionEvaluations { get; private set; } + public int FunctionEvaluations { get; set; } /// /// Get the number of calls to jacobian. /// - public int JacobianEvaluations { get; private set; } + public int JacobianEvaluations { get; set; } /// /// Set or get the desired accuracy order of the numerical jacobian. diff --git a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs index a8ab570f..85d4fef0 100644 --- a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs +++ b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs @@ -121,6 +121,10 @@ namespace MathNet.Numerics.Optimization ExitCondition exitCondition = ExitCondition.None; + // Initialize objective + objective.FunctionEvaluations = 0; + objective.JacobianEvaluations = 0; + // First, calculate function values and setup variables objective.EvaluateFunction(initialGuess); var P = objective.Parameters; // current parameters From ef5e93697ed5f45733e5c574907d331f63e673b1 Mon Sep 17 00:00:00 2001 From: diluculo Date: Tue, 1 Jan 2019 01:55:36 +0900 Subject: [PATCH 04/13] Reworked TrustRegionMinimizer to support various subproblems. --- .../Optimization/ITrustRegionSubProblem.cs | 13 + .../Subproblems/QuadraticSubproblem.cs | 54 +++ src/Numerics/Optimization/Subproblems/Util.cs | 32 ++ .../TrustRegionDogLegMinimizer.cs | 320 +----------------- .../Optimization/TrustRegionMinimizerBase.cs | 261 ++++++++++++++ .../Optimization/TrustRegionSubProblem.cs | 12 + 6 files changed, 375 insertions(+), 317 deletions(-) create mode 100644 src/Numerics/Optimization/ITrustRegionSubProblem.cs create mode 100644 src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs create mode 100644 src/Numerics/Optimization/Subproblems/Util.cs create mode 100644 src/Numerics/Optimization/TrustRegionMinimizerBase.cs create mode 100644 src/Numerics/Optimization/TrustRegionSubProblem.cs diff --git a/src/Numerics/Optimization/ITrustRegionSubProblem.cs b/src/Numerics/Optimization/ITrustRegionSubProblem.cs new file mode 100644 index 00000000..7e2d7e07 --- /dev/null +++ b/src/Numerics/Optimization/ITrustRegionSubProblem.cs @@ -0,0 +1,13 @@ +using MathNet.Numerics.LinearAlgebra; + +namespace MathNet.Numerics.Optimization +{ + public interface ITrustRegionSubproblem + { + Vector Pstep { get; } + double PredictedReduction { get; } + bool HitBoundary { get; } + + void Solve(IObjectiveModel objective, double radius); + } +} diff --git a/src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs b/src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs new file mode 100644 index 00000000..499982eb --- /dev/null +++ b/src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs @@ -0,0 +1,54 @@ +using MathNet.Numerics.LinearAlgebra; + +namespace MathNet.Numerics.Optimization.Subproblems +{ + internal class QuadraticSubproblem : ITrustRegionSubproblem + { + public Vector Pstep { get; private set; } + + public double PredictedReduction { get; private set; } + + public bool HitBoundary { get; private set; } + + public void Solve(IObjectiveModel objective, double delta) + { + var Jacobian = objective.Jacobian; + var Gradient = objective.Gradient; + var Hessian = objective.Hessian; + var RSS = objective.Residue; + + // newton point + // the Gauss–Newton step by solving the normal equations + var Pgn = Hessian.PseudoInverse() * Gradient; // Hessian.Solve(Gradient) fails so many times... + + // cauchy point + // steepest descent direction is given by + var alpha = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); + var Psd = alpha * Gradient; + + // update step and prectted reduction + if (Pgn.L2Norm() <= delta) + { + // Pgn is inside trust region radius + HitBoundary = false; + Pstep = Pgn; + PredictedReduction = RSS; + } + else if (alpha * Psd.L2Norm() >= delta) + { + // Psd is outside trust region radius + HitBoundary = true; + Pstep = delta / Psd.L2Norm() * Psd; + PredictedReduction = delta * (2.0 * (alpha * Gradient).L2Norm() - delta) / 2.0 / alpha; + } + else + { + // Pstep is intersection of the trust region boundary + HitBoundary = true; + var beta = Util.FindBeta(alpha, Psd, Pgn, delta).Item2; + Pstep = alpha * Psd + beta * (Pgn - alpha * Psd); + PredictedReduction = 0.5 * alpha * (1 - beta) * (1 - beta) * Gradient.DotProduct(Gradient) + beta * (2 - beta) * RSS; + } + } + } +} diff --git a/src/Numerics/Optimization/Subproblems/Util.cs b/src/Numerics/Optimization/Subproblems/Util.cs new file mode 100644 index 00000000..c15f636b --- /dev/null +++ b/src/Numerics/Optimization/Subproblems/Util.cs @@ -0,0 +1,32 @@ +using MathNet.Numerics.LinearAlgebra; +using System; + +namespace MathNet.Numerics.Optimization.Subproblems +{ + internal static class Util + { + public static Tuple FindBeta(double alpha, Vector sd, Vector gn, double delta) + { + // Pstep is intersection of the trust region boundary + // Pstep = α*Psd + β*(Pgn - α*Psd) + // find r so that ||Pstep|| = Δ + // z = α*Psd, d = (Pgn - z) + // (d^2)β^2 + (2*z*d)β + (z^2 - Δ^2) = 0 + // + // positive β is used for the quadratic formula + + var z = alpha * sd; + var d = gn - z; + + var a = d.DotProduct(d); + var b = 2.0 * z.DotProduct(d); + var c = z.DotProduct(z) - delta * delta; + + var aux = b + ((b >= 0) ? 1.0 : -1.0) * Math.Sqrt(b * b - 4.0 * a * c); + var beta1 = -aux / 2.0 / a; + var beta2 = -2.0 * c / aux; + + return new Tuple(beta1, beta2); + } + } +} diff --git a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs index 85d4fef0..af4efc4d 100644 --- a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs +++ b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs @@ -3,324 +3,10 @@ using System; namespace MathNet.Numerics.Optimization { - public sealed class TrustRegionDogLegMinimizer + public sealed class TrustRegionDogLegMinimizer : TrustRegionMinimizerBase { - #region Tolerances and options - - /// - /// The stopping threshold for infinity norm of the gradient. - /// - public static double GradientTolerance { get; set; } - - /// - /// The stopping threshold for L2 norm of the change of the parameters. - /// - public static double StepTolerance { get; set; } - - /// - /// The stopping threshold for the function value or L2 norm of the residuals. - /// - public static double FunctionTolerance { get; set; } - - /// - /// The stopping threshold for the trust region radius. - /// - public static double RadiusTolerance { get; set; } - - /// - /// The maximum number of iterations. - /// - public int MaximumIterations { get; set; } - - #endregion Tolerances and options - public TrustRegionDogLegMinimizer(double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) - { - FunctionTolerance = functionTolerance; - GradientTolerance = gradientTolerance; - StepTolerance = stepTolerance; - RadiusTolerance = radiusTolerance; - MaximumIterations = maximumIterations; - } - - public ModelMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) - { - if (objective == null) - throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - - return Minimum(objective, initialGuess, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); - } - - public ModelMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) - { - if (objective == null) - throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - - return Minimum(objective, CreateVector.DenseOfArray(initialGuess), GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); - } - - /// - /// Non-linear least square fitting by the trust-region dogleg algorithm. - /// - /// The objective model, including function, jacobian, observations, and parameter bounds. - /// The initial guess values. - /// The stopping threshold for L2 norm of the residuals. - /// The stopping threshold for infinity norm of the gradient vector. - /// The stopping threshold for L2 norm of the change of parameters. - /// The stopping threshold for trust region radius - /// The max iterations. - /// - public static ModelMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) - { - // Non-linear least square fitting by the trust-region dogleg algorithm. - // - // The Powell's dogleg method is finding the minimum of a function F(p) that is a sum of squares of nonlinear functions. - // - // For given datum pair (x, y), uncertainties σ (or weighting W = 1 / σ^2) and model function f = f(x; p), - // let's find the parameters of the model so that the sum of the quares of the deviations is minimized. - // - // F(p) = 1/2 * ∑{ Wi * (yi - f(xi; p))^2 } - // pbest = argmin F(p) - // - // Here, we will use the following terms: - // Weighting W is the diagonal matrix and can be decomposed as LL', so L = 1/σ - // Residuals, R = L(y - f(x; p)) - // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) - // Jacobian J = df(x; p)/dp - // Gradient g = J'W(y − f(x; p)) = J'LR - // Approximated Hessian H = J'WJ - // - // The Powell's dogleg algorithm is summarized as follows: - // initially set trust-region radius, Δ - // repeat - // solve quadratic subproblem - // update Δ: - // let ρ = (RSS - RSSnew) / predRed - // if ρ > 0.75, Δ = 2Δ - // if ρ < 0.25, Δ = Δ/4 - // if ρ > eta, P = P + ΔP - // - // References: - // [1]. Madsen, K., H. B. Nielsen, and O. Tingleff. - // "Methods for Non-Linear Least Squares Problems. Technical University of Denmark, 2004. Lecture notes." (2004). - // Available Online from: http://orbit.dtu.dk/files/2721358/imm3215.pdf - // [2]. SciPy - // Available Online from: https://github.com/scipy/scipy/blob/master/scipy/optimize/_trustregion_dogleg.py - - double maxDelta = 1000; - double eta = 0; - - if (objective == null) - throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - - ExitCondition exitCondition = ExitCondition.None; - - // Initialize objective - objective.FunctionEvaluations = 0; - objective.JacobianEvaluations = 0; - - // First, calculate function values and setup variables - objective.EvaluateFunction(initialGuess); - var P = objective.Parameters; // current parameters - var RSS = objective.Residue; // Residual Sum of Squares = R'R - var RSSinit = RSS; // RSS at initial gussing parameters - - if (maximumIterations < 0) - { - maximumIterations = 200 * (initialGuess.Count + 1); - } - - // if R == NaN, stop - if (double.IsNaN(RSS)) - { - exitCondition = ExitCondition.InvalidValues; - return new ModelMinimizationResult(objective, -1, exitCondition); - } - - // When only function evaluation is needed, set maximumIterations to zero, - if (maximumIterations == 0) - { - exitCondition = ExitCondition.ManuallyStopped; - } - - // if ||R||^2 <= fTol, stop - if (RSS <= functionTolerance) - { - exitCondition = ExitCondition.Converged; // SmallRSS - } - - // Evaluate projected Hessian, and gradient - objective.EvaluateJacobian(P); - var Hessian = objective.Hessian; - var Gradient = objective.Gradient; - - // if ||g||_oo <= gtol, found and stop - if (Gradient.InfinityNorm() <= gradientTolerance) - { - exitCondition = ExitCondition.RelativeGradient; // SmallGradient - } - - if (exitCondition != ExitCondition.None) - { - // finalize - objective.EvaluateCovariance(P); - return new ModelMinimizationResult(objective, -1, exitCondition); - } - - // initialize trust-region radius, Δ - double delta = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); - delta = Math.Max(1.0, Math.Min(delta, maxDelta)); - - int iterations = 0; - while (iterations < maximumIterations && exitCondition == ExitCondition.None) - { - iterations++; - - // solve the subproblem - var subprogram = SolveQuadraticSubproblem(objective, delta); - var Pstep = subprogram.Item1; - var predictedReduction = subprogram.Item2; - var hitBoundary = subprogram.Item3; - - if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) - { - exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters - break; - } - - var Pnew = P + Pstep; // parameters to test - - objective.EvaluateFunction(Pnew); - var RSSnew = objective.Residue; - - // calculate the ratio of the actual to the predicted reduction. - double rho = (predictedReduction != 0) - ? (RSS - RSSnew) / predictedReduction - : 0; - - if (rho > 0.75 && hitBoundary) - { - delta = Math.Min(2.0 * delta, maxDelta); - } - else if (rho < 0.25) - { - delta = delta * 0.25; - if (delta <= radiusTolerance * (radiusTolerance + P.DotProduct(P))) - { - exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters - break; - } - } - - if (rho > eta) - { - // accepted - Pnew.CopyTo(P); - RSS = RSSnew; - - // update Jacobian, Hessian, and gradient - objective.EvaluateJacobian(P); - Gradient = objective.Gradient; - Hessian = objective.Hessian; - - // if ||g||_oo <= gtol, found and stop - if (Gradient.InfinityNorm() <= gradientTolerance) - { - exitCondition = ExitCondition.RelativeGradient; - } - - // if ||R||^2 < fTol, found and stop - if (RSS <= functionTolerance) - { - exitCondition = ExitCondition.Converged; // SmallRSS - } - } - } - - if (iterations >= maximumIterations) - { - exitCondition = ExitCondition.ExceedIterations; - } - - // finalize - objective.EvaluateCovariance(P); - - return new ModelMinimizationResult(objective, iterations, exitCondition); - } - - private static Tuple, double, bool> SolveQuadraticSubproblem(IObjectiveModel objective, double delta) - { - Vector Pstep; - double predictedReduction; - bool hitBoundary = false; - - var Jacobian = objective.Jacobian; - var Gradient = objective.Gradient; - var Hessian = objective.Hessian; - var RSS = objective.Residue; - - // newton point - // the Gauss–Newton step by solving the normal equations - var Pgn = Hessian.PseudoInverse() * Gradient; // Hessian.Solve(Gradient) fails so many times... - - // cauchy point - // steepest descent direction is given by - var alpha = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); - var Psd = alpha * Gradient; - - // update step and prectted reduction - if (Pgn.L2Norm() <= delta) - { - // Pgn is inside trust region radius - hitBoundary = false; - Pstep = Pgn; - predictedReduction = RSS; - } - else if (alpha * Psd.L2Norm() >= delta) - { - // Psd is outside trust region radius - hitBoundary = true; - Pstep = delta / Psd.L2Norm() * Psd; - predictedReduction = delta * (2.0 * (alpha * Gradient).L2Norm() - delta) / 2.0 / alpha; - } - else - { - // Pstep is intersection of the trust region boundary - hitBoundary = true; - var beta = FindBeta(alpha, Psd, Pgn, delta); - Pstep = alpha * Psd + beta * (Pgn - alpha * Psd); - predictedReduction = 0.5 * alpha * (1 - beta) * (1 - beta) * Gradient.DotProduct(Gradient) + beta * (2 - beta) * RSS; - } - - return new Tuple, double, bool>(Pstep, predictedReduction, hitBoundary); - } - - private static double FindBeta(double alpha, Vector sd, Vector gn, double delta) - { - // Pstep is intersection of the trust region boundary - // Pstep = α*Psd + β*(Pgn - α*Psd) - // find r so that ||Pstep|| = Δ - // z = α*Psd, d = (Pgn - z) - // (d^2)β^2 + (2*z*d)β + (z^2 - Δ^2) = 0 - // get positive β by using the quadratic formula - - var z = alpha * sd; - var d = gn - z; - - var a = d.DotProduct(d); - var b = 2.0 * z.DotProduct(d); - var c = z.DotProduct(z) - delta * delta; - - var aux = b + ((b >= 0) ? 1.0 : -1.0) * Math.Sqrt(b * b - 4.0 * a * c); - var beta = Math.Max(-aux / 2.0 / a, -2.0 * c / aux); - - return beta; - } + : base(TrustRegionSubproblem.Quadratic(), gradientTolerance, stepTolerance, functionTolerance, radiusTolerance, maximumIterations) + { } } } diff --git a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs new file mode 100644 index 00000000..4bfc315d --- /dev/null +++ b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs @@ -0,0 +1,261 @@ +using MathNet.Numerics.LinearAlgebra; +using System; + +namespace MathNet.Numerics.Optimization +{ + public abstract class TrustRegionMinimizerBase + { + public static ITrustRegionSubproblem Subproblem; + + /// + /// The stopping threshold for infinity norm of the gradient. + /// + public static double GradientTolerance { get; set; } + + /// + /// The stopping threshold for L2 norm of the change of the parameters. + /// + public static double StepTolerance { get; set; } + + /// + /// The stopping threshold for the function value or L2 norm of the residuals. + /// + public static double FunctionTolerance { get; set; } + + /// + /// The stopping threshold for the trust region radius. + /// + public static double RadiusTolerance { get; set; } + + /// + /// The maximum number of iterations. + /// + public int MaximumIterations { get; set; } + + public TrustRegionMinimizerBase(ITrustRegionSubproblem subproblem, + double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) + { + if (subproblem == null) + throw new ArgumentNullException("subproblem"); + + Subproblem = subproblem; + FunctionTolerance = functionTolerance; + GradientTolerance = gradientTolerance; + StepTolerance = stepTolerance; + RadiusTolerance = radiusTolerance; + MaximumIterations = maximumIterations; + } + + public ModelMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + { + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + return Minimum(objective, initialGuess, Subproblem, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); + } + + public ModelMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + { + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + return Minimum(objective, CreateVector.DenseOfArray(initialGuess), Subproblem, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); + } + + /// + /// Non-linear least square fitting by the trust-region algorithm. + /// + /// The objective model, including function, jacobian, observations, and parameter bounds. + /// The initial guess values. + /// The subproblem + /// The stopping threshold for L2 norm of the residuals. + /// The stopping threshold for infinity norm of the gradient vector. + /// The stopping threshold for L2 norm of the change of parameters. + /// The stopping threshold for trust region radius + /// The max iterations. + /// + public static ModelMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, ITrustRegionSubproblem subproblem, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) + { + // Non-linear least square fitting by the trust-region algorithm. + // + // For given datum pair (x, y), uncertainties σ (or weighting W = 1 / σ^2) and model function f = f(x; p), + // let's find the parameters of the model so that the sum of the quares of the deviations is minimized. + // + // F(p) = 1/2 * ∑{ Wi * (yi - f(xi; p))^2 } + // pbest = argmin F(p) + // + // Here, we will use the following terms: + // Weighting W is the diagonal matrix and can be decomposed as LL', so L = 1/σ + // Residuals, R = L(y - f(x; p)) + // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) + // Jacobian J = df(x; p)/dp + // Gradient g = J'W(y − f(x; p)) = J'LR + // Approximated Hessian H = J'WJ + // + // The trust region algorithm is summarized as follows: + // initially set trust-region radius, Δ + // repeat + // solve subproblem + // update Δ: + // let ρ = (RSS - RSSnew) / predRed + // if ρ > 0.75, Δ = 2Δ + // if ρ < 0.25, Δ = Δ/4 + // if ρ > eta, P = P + ΔP + // + // References: + // [1]. Madsen, K., H. B. Nielsen, and O. Tingleff. + // "Methods for Non-Linear Least Squares Problems. Technical University of Denmark, 2004. Lecture notes." (2004). + // Available Online from: http://orbit.dtu.dk/files/2721358/imm3215.pdf + // [2]. Nocedal, Jorge, and Stephen J. Wright. + // Numerical optimization (2006): 101-134. + // [3]. SciPy + // Available Online from: https://github.com/scipy/scipy/blob/master/scipy/optimize/_trustregion.py + + double maxDelta = 1000; + double eta = 0; + + if (objective == null) + throw new ArgumentNullException("objective"); + if (initialGuess == null) + throw new ArgumentNullException("initialGuess"); + + ExitCondition exitCondition = ExitCondition.None; + + // Initialize objective + objective.FunctionEvaluations = 0; + objective.JacobianEvaluations = 0; + + // First, calculate function values and setup variables + objective.EvaluateFunction(initialGuess); + var P = objective.Parameters; // current parameters + var RSS = objective.Residue; // Residual Sum of Squares = R'R + var RSSinit = RSS; // RSS at initial gussing parameters + + if (maximumIterations < 0) + { + maximumIterations = 200 * (initialGuess.Count + 1); + } + + // if R == NaN, stop + if (double.IsNaN(RSS)) + { + exitCondition = ExitCondition.InvalidValues; + return new ModelMinimizationResult(objective, -1, exitCondition); + } + + // When only function evaluation is needed, set maximumIterations to zero, + if (maximumIterations == 0) + { + exitCondition = ExitCondition.ManuallyStopped; + } + + // if ||R||^2 <= fTol, stop + if (RSS <= functionTolerance) + { + exitCondition = ExitCondition.Converged; // SmallRSS + } + + // Evaluate projected Hessian, and gradient + objective.EvaluateJacobian(P); + var Hessian = objective.Hessian; + var Gradient = objective.Gradient; + + // if ||g||_oo <= gtol, found and stop + if (Gradient.InfinityNorm() <= gradientTolerance) + { + exitCondition = ExitCondition.RelativeGradient; // SmallGradient + } + + if (exitCondition != ExitCondition.None) + { + // finalize + objective.EvaluateCovariance(P); + return new ModelMinimizationResult(objective, -1, exitCondition); + } + + // initialize trust-region radius, Δ + double delta = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); + delta = Math.Max(1.0, Math.Min(delta, maxDelta)); + + int iterations = 0; + while (iterations < maximumIterations && exitCondition == ExitCondition.None) + { + iterations++; + + // solve the subproblem + subproblem.Solve(objective, delta); + var Pstep = subproblem.Pstep; + var predictedReduction = subproblem.PredictedReduction; + var hitBoundary = subproblem.HitBoundary; + + if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) + { + exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + break; + } + + var Pnew = P + Pstep; // parameters to test + + objective.EvaluateFunction(Pnew); + var RSSnew = objective.Residue; + + // calculate the ratio of the actual to the predicted reduction. + double rho = (predictedReduction != 0) + ? (RSS - RSSnew) / predictedReduction + : 0; + + if (rho > 0.75 && hitBoundary) + { + delta = Math.Min(2.0 * delta, maxDelta); + } + else if (rho < 0.25) + { + delta = delta * 0.25; + if (delta <= radiusTolerance * (radiusTolerance + P.DotProduct(P))) + { + exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + break; + } + } + + if (rho > eta) + { + // accepted + Pnew.CopyTo(P); + RSS = RSSnew; + + // update Jacobian, Hessian, and gradient + objective.EvaluateJacobian(P); + Gradient = objective.Gradient; + Hessian = objective.Hessian; + + // if ||g||_oo <= gtol, found and stop + if (Gradient.InfinityNorm() <= gradientTolerance) + { + exitCondition = ExitCondition.RelativeGradient; + } + + // if ||R||^2 < fTol, found and stop + if (RSS <= functionTolerance) + { + exitCondition = ExitCondition.Converged; // SmallRSS + } + } + } + + if (iterations >= maximumIterations) + { + exitCondition = ExitCondition.ExceedIterations; + } + + // finalize + objective.EvaluateCovariance(P); + + return new ModelMinimizationResult(objective, iterations, exitCondition); + } + } +} diff --git a/src/Numerics/Optimization/TrustRegionSubProblem.cs b/src/Numerics/Optimization/TrustRegionSubProblem.cs new file mode 100644 index 00000000..aed6886b --- /dev/null +++ b/src/Numerics/Optimization/TrustRegionSubProblem.cs @@ -0,0 +1,12 @@ +using MathNet.Numerics.Optimization.Subproblems; + +namespace MathNet.Numerics.Optimization +{ + public static class TrustRegionSubproblem + { + public static ITrustRegionSubproblem Quadratic() + { + return new QuadraticSubproblem(); + } + } +} From 04b8b78dbae034e3ef063d01eaa9524140fbe894 Mon Sep 17 00:00:00 2001 From: diluculo Date: Wed, 2 Jan 2019 09:56:37 +0900 Subject: [PATCH 05/13] Renamed QuadraticSubproblem to DogLegSubproblem. --- src/Numerics/Optimization/ITrustRegionSubProblem.cs | 1 - .../{QuadraticSubproblem.cs => DogLegSubproblem.cs} | 7 +------ src/Numerics/Optimization/Subproblems/Util.cs | 5 ++++- src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs | 2 +- src/Numerics/Optimization/TrustRegionMinimizerBase.cs | 3 ++- src/Numerics/Optimization/TrustRegionSubProblem.cs | 4 ++-- 6 files changed, 10 insertions(+), 12 deletions(-) rename src/Numerics/Optimization/Subproblems/{QuadraticSubproblem.cs => DogLegSubproblem.cs} (80%) diff --git a/src/Numerics/Optimization/ITrustRegionSubProblem.cs b/src/Numerics/Optimization/ITrustRegionSubProblem.cs index 7e2d7e07..5305fc44 100644 --- a/src/Numerics/Optimization/ITrustRegionSubProblem.cs +++ b/src/Numerics/Optimization/ITrustRegionSubProblem.cs @@ -5,7 +5,6 @@ namespace MathNet.Numerics.Optimization public interface ITrustRegionSubproblem { Vector Pstep { get; } - double PredictedReduction { get; } bool HitBoundary { get; } void Solve(IObjectiveModel objective, double radius); diff --git a/src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs similarity index 80% rename from src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs rename to src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs index 499982eb..e3f06d15 100644 --- a/src/Numerics/Optimization/Subproblems/QuadraticSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs @@ -2,12 +2,10 @@ namespace MathNet.Numerics.Optimization.Subproblems { - internal class QuadraticSubproblem : ITrustRegionSubproblem + internal class DogLegSubproblem : ITrustRegionSubproblem { public Vector Pstep { get; private set; } - public double PredictedReduction { get; private set; } - public bool HitBoundary { get; private set; } public void Solve(IObjectiveModel objective, double delta) @@ -32,14 +30,12 @@ namespace MathNet.Numerics.Optimization.Subproblems // Pgn is inside trust region radius HitBoundary = false; Pstep = Pgn; - PredictedReduction = RSS; } else if (alpha * Psd.L2Norm() >= delta) { // Psd is outside trust region radius HitBoundary = true; Pstep = delta / Psd.L2Norm() * Psd; - PredictedReduction = delta * (2.0 * (alpha * Gradient).L2Norm() - delta) / 2.0 / alpha; } else { @@ -47,7 +43,6 @@ namespace MathNet.Numerics.Optimization.Subproblems HitBoundary = true; var beta = Util.FindBeta(alpha, Psd, Pgn, delta).Item2; Pstep = alpha * Psd + beta * (Pgn - alpha * Psd); - PredictedReduction = 0.5 * alpha * (1 - beta) * (1 - beta) * Gradient.DotProduct(Gradient) + beta * (2 - beta) * RSS; } } } diff --git a/src/Numerics/Optimization/Subproblems/Util.cs b/src/Numerics/Optimization/Subproblems/Util.cs index c15f636b..4ea986f8 100644 --- a/src/Numerics/Optimization/Subproblems/Util.cs +++ b/src/Numerics/Optimization/Subproblems/Util.cs @@ -26,7 +26,10 @@ namespace MathNet.Numerics.Optimization.Subproblems var beta1 = -aux / 2.0 / a; var beta2 = -2.0 * c / aux; - return new Tuple(beta1, beta2); + // return sorted beta + return (beta1 < beta2) + ? new Tuple(beta1, beta2) + : new Tuple(beta2, beta1); } } } diff --git a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs index af4efc4d..596f7637 100644 --- a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs +++ b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs @@ -6,7 +6,7 @@ namespace MathNet.Numerics.Optimization public sealed class TrustRegionDogLegMinimizer : TrustRegionMinimizerBase { public TrustRegionDogLegMinimizer(double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) - : base(TrustRegionSubproblem.Quadratic(), gradientTolerance, stepTolerance, functionTolerance, radiusTolerance, maximumIterations) + : base(TrustRegionSubproblem.DogLeg(), gradientTolerance, stepTolerance, functionTolerance, radiusTolerance, maximumIterations) { } } } diff --git a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs index 4bfc315d..f4125e8f 100644 --- a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs +++ b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs @@ -189,8 +189,9 @@ namespace MathNet.Numerics.Optimization // solve the subproblem subproblem.Solve(objective, delta); var Pstep = subproblem.Pstep; - var predictedReduction = subproblem.PredictedReduction; var hitBoundary = subproblem.HitBoundary; + // predicted reduction = L(0) - L(Δp) = Δp'g - 1/2 * Δp'HΔp + var predictedReduction = objective.Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(objective.Hessian * Pstep); if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) { diff --git a/src/Numerics/Optimization/TrustRegionSubProblem.cs b/src/Numerics/Optimization/TrustRegionSubProblem.cs index aed6886b..cde8ecf8 100644 --- a/src/Numerics/Optimization/TrustRegionSubProblem.cs +++ b/src/Numerics/Optimization/TrustRegionSubProblem.cs @@ -4,9 +4,9 @@ namespace MathNet.Numerics.Optimization { public static class TrustRegionSubproblem { - public static ITrustRegionSubproblem Quadratic() + public static ITrustRegionSubproblem DogLeg() { - return new QuadraticSubproblem(); + return new DogLegSubproblem(); } } } From bf4b901fa5fa31779b963632a238ebac5f845147 Mon Sep 17 00:00:00 2001 From: diluculo Date: Wed, 2 Jan 2019 15:33:34 +0900 Subject: [PATCH 06/13] Optimization: added trust region Newton-CG minimizer. --- .../NonLinearCurveFittingTests.cs | 62 ++++++++++++++++-- .../Subproblems/DogLegSubproblem.cs | 1 - .../Subproblems/NewtonCGSubproblem.cs | 65 +++++++++++++++++++ .../TrustRegionDogLegMinimizer.cs | 8 +-- .../TrustRegionNewtonCGMinimizer.cs | 13 ++++ .../Optimization/TrustRegionSubProblem.cs | 5 ++ 6 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs create mode 100644 src/Numerics/Optimization/TrustRegionNewtonCGMinimizer.cs diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs index 4918a263..f0787922 100644 --- a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -93,8 +93,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRLMDER_FindMinimum_Rosenbrock_Unconstrained() + public void TRDLDER_FindMinimum_Rosenbrock_Unconstrained() { + // DogLeg Minimizer var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); var result = solver.FindMinimum(obj, RosenbrockStart1); @@ -103,10 +104,20 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); } + + // NewtonCG Minimizer + obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + var solverNCG = new TrustRegionNewtonCGMinimizer(maximumIterations: 10000); + result = solverNCG.FindMinimum(obj, RosenbrockStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); + } } [Test] - public void TRLMDIF_FindMinimum_Rosenbrock_Unconstrained() + public void TRDLDIF_FindMinimum_Rosenbrock_Unconstrained() { var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); @@ -118,6 +129,19 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + [Test] + public void TRNCGDER_FindMinimum_Rosenbrock_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + var solver = new TrustRegionNewtonCGMinimizer(maximumIterations: 10000); + var result = solver.FindMinimum(obj, RosenbrockStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); + } + } + // model: Rat43 (https://www.itl.nist.gov/div898/strd/nls/data/ratkowsky3.shtml) // f(x; a, b, c, d) = a / ((1 + exp(b - c * x))^(1 / d)) // best fitted parameters: @@ -335,7 +359,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRLMDIF_FindMinimum_BoxBod_Unconstrained() + public void TRDLDIF_FindMinimum_BoxBod_Unconstrained() { var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); @@ -348,6 +372,20 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + [Test] + public void TRNCGDIF_FindMinimum_BoxBod_Unconstrained() + { + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); + var solver = new TrustRegionNewtonCGMinimizer(); + var result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 3); + } + } + // model : Thurber (https://www.itl.nist.gov/div898/strd/nls/data/thurber.shtml) // f(x; b1 ... b7) = (b1 + b2*x + b3*x^2 + b4*x^3) / (1 + b5*x + b6*x^2 + b7*x^3) // derivatives: @@ -450,7 +488,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRLMDIF_FindMinimum_Thurber_Scaled() + public void TRDLDIF_FindMinimum_Thurber_Scaled() { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, scales: ThurberScales, @@ -464,5 +502,21 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 3); } } + + [Test] + public void TRNCGDIF_FindMinimum_Thurber_Scaled() + { + var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, + scales: ThurberScales, + accuracyOrder: 6); + var solver = new TrustRegionNewtonCGMinimizer(); + var result = solver.FindMinimum(obj, ThurberInitialGuess); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 3); + } + } } } diff --git a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs index e3f06d15..8d41288d 100644 --- a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs @@ -13,7 +13,6 @@ namespace MathNet.Numerics.Optimization.Subproblems var Jacobian = objective.Jacobian; var Gradient = objective.Gradient; var Hessian = objective.Hessian; - var RSS = objective.Residue; // newton point // the Gauss–Newton step by solving the normal equations diff --git a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs new file mode 100644 index 00000000..ba6f9a6c --- /dev/null +++ b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs @@ -0,0 +1,65 @@ +using MathNet.Numerics.LinearAlgebra; +using System; + +namespace MathNet.Numerics.Optimization.Subproblems +{ + internal class NewtonCGSubproblem : ITrustRegionSubproblem + { + public Vector Pstep { get; private set; } + + public bool HitBoundary { get; private set; } + + public void Solve(IObjectiveModel objective, double delta) + { + var Jacobian = objective.Jacobian; + var Gradient = objective.Gradient; + var Hessian = objective.Hessian; + + // define tolerance + var tolerance = Math.Min(0.5, Math.Sqrt(Gradient.L2Norm())) * Gradient.L2Norm(); + + // initialize internal variables + var z = Vector.Build.Dense(Hessian.RowCount); + var r = -Gradient; + var d = -r; + + while (true) + { + var Bd = Hessian * d; + var dBd = d.DotProduct(Bd); + + if (dBd <= 0) + { + var t = Util.FindBeta(1, z, d, delta); + Pstep = z + t.Item1 * d; + HitBoundary = true; + return; + } + + var r_sq = r.DotProduct(r); + var alpha = r_sq / dBd; + var znext = z + alpha * d; + if(znext.L2Norm() >= delta) + { + var t = Util.FindBeta(1, z, d, delta); + Pstep = z + t.Item2 * d; + HitBoundary = true; + return; + } + + var rnext = r + alpha * Bd; + var rnext_sq = rnext.DotProduct(rnext); + if (Math.Sqrt(rnext_sq) < tolerance) + { + Pstep = znext; + HitBoundary = false; + return; + } + + z = znext; + r = rnext; + d = -rnext + rnext_sq / r_sq * d; ; + } + } + } +} diff --git a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs index 596f7637..77a220f0 100644 --- a/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs +++ b/src/Numerics/Optimization/TrustRegionDogLegMinimizer.cs @@ -1,10 +1,10 @@ -using MathNet.Numerics.LinearAlgebra; -using System; - -namespace MathNet.Numerics.Optimization +namespace MathNet.Numerics.Optimization { public sealed class TrustRegionDogLegMinimizer : TrustRegionMinimizerBase { + /// + /// Non-linear least square fitting by the trust region dogleg algorithm. + /// public TrustRegionDogLegMinimizer(double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) : base(TrustRegionSubproblem.DogLeg(), gradientTolerance, stepTolerance, functionTolerance, radiusTolerance, maximumIterations) { } diff --git a/src/Numerics/Optimization/TrustRegionNewtonCGMinimizer.cs b/src/Numerics/Optimization/TrustRegionNewtonCGMinimizer.cs new file mode 100644 index 00000000..7e6d5f5a --- /dev/null +++ b/src/Numerics/Optimization/TrustRegionNewtonCGMinimizer.cs @@ -0,0 +1,13 @@ +namespace MathNet.Numerics.Optimization +{ + public sealed class TrustRegionNewtonCGMinimizer : TrustRegionMinimizerBase + { + /// + /// Non-linear least square fitting by the trust region Newton-Conjugate-Gradient algorithm. + /// + public TrustRegionNewtonCGMinimizer(double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) + : base(TrustRegionSubproblem.NewtonCG(), gradientTolerance, stepTolerance, functionTolerance, radiusTolerance, maximumIterations) + { } + } +} + diff --git a/src/Numerics/Optimization/TrustRegionSubProblem.cs b/src/Numerics/Optimization/TrustRegionSubProblem.cs index cde8ecf8..3018f8b6 100644 --- a/src/Numerics/Optimization/TrustRegionSubProblem.cs +++ b/src/Numerics/Optimization/TrustRegionSubProblem.cs @@ -8,5 +8,10 @@ namespace MathNet.Numerics.Optimization { return new DogLegSubproblem(); } + + public static ITrustRegionSubproblem NewtonCG() + { + return new NewtonCGSubproblem(); + } } } From 67db4f0501878bc9046a0a5e41559e9f015c98de Mon Sep 17 00:00:00 2001 From: diluculo Date: Wed, 2 Jan 2019 17:41:22 +0900 Subject: [PATCH 07/13] Added a converter from IObjectiveModel to IObjectiveFunction. --- .../NonLinearCurveFittingTests.cs | 64 +++++++++++++++++-- src/Numerics/Optimization/IObjectiveModel.cs | 2 + src/Numerics/Optimization/ObjectiveModel.cs | 24 +++++++ .../ObjectiveModels/FittingObjectiveModel.cs | 23 ++++++- 4 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs index f0787922..c353b83b 100644 --- a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -386,6 +386,19 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + [Test] + public void Bfgs_FindMinimum_BoxBod_Unconstrained() + { + var obj = ObjectiveModel.FittingFunction(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 100); + var result = solver.FindMinimum(obj, BoxBodStart2); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); + } + } + // model : Thurber (https://www.itl.nist.gov/div898/strd/nls/data/thurber.shtml) // f(x; b1 ... b7) = (b1 + b2*x + b3*x^2 + b4*x^3) / (1 + b5*x + b6*x^2 + b7*x^3) // derivatives: @@ -456,7 +469,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector ThurberPstd = new DenseVector(new double[] { 4.6647963344E+00, 3.9571156086E+01, 2.8698696102E+01, 5.5675370270E+00, 3.1333340687E-02, 1.4984928198E-02, 6.5842344623E-03 }); - private Vector ThurberInitialGuess = new DenseVector(new double[] { 1000.0, 1000.0, 400.0, 40.0, 0.7, 0.3, 0.03 }); + private Vector ThurberStart = new DenseVector(new double[] { 1000.0, 1000.0, 400.0, 40.0, 0.7, 0.3, 0.03 }); + private Vector ThurberLowerBound = new DenseVector(new double[] { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }); + private Vector ThurberUpperBound = new DenseVector(new double[] { 1E6, 1E6, 1E6, 1E6, 1E6, 1E6, 1E6 }); private Vector ThurberScales = new DenseVector(new double[7] { 1000, 1000, 400, 40, 0.7, 0.3, 0.03 }); [Test] @@ -464,7 +479,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberPrime, ThurberX, ThurberY); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); + var result = solver.FindMinimum(obj, ThurberStart); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -478,7 +493,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); + var result = solver.FindMinimum(obj, ThurberStart); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -494,7 +509,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests scales: ThurberScales, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); + var result = solver.FindMinimum(obj, ThurberStart); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -510,7 +525,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests scales: ThurberScales, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); - var result = solver.FindMinimum(obj, ThurberInitialGuess); + var result = solver.FindMinimum(obj, ThurberStart); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -518,5 +533,44 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 3); } } + + [Test] + public void Bfgs_FindMinimum_Thurber_Unconstrained() + { + var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); + var result = solver.FindMinimum(obj, ThurberStart); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); + } + } + + [Test] + public void BfgsB_FindMinimum_Thurber() + { + var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var solver = new BfgsBMinimizer(1e-10, 1e-10, 1e-10, 1000); + var result = solver.FindMinimum(obj, ThurberLowerBound, ThurberUpperBound, ThurberStart); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); + } + } + + [Test] + public void LBfgs_FindMinimum_Thurber() + { + var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); + var result = solver.FindMinimum(obj, ThurberStart); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); + } + } } } diff --git a/src/Numerics/Optimization/IObjectiveModel.cs b/src/Numerics/Optimization/IObjectiveModel.cs index 72c15391..2409dffe 100644 --- a/src/Numerics/Optimization/IObjectiveModel.cs +++ b/src/Numerics/Optimization/IObjectiveModel.cs @@ -66,5 +66,7 @@ namespace MathNet.Numerics.Optimization /// Create a new independent copy of this objective function, evaluated at the same point. IObjectiveModel Fork(); + + IObjectiveFunction ToObjectiveFunction(); } } diff --git a/src/Numerics/Optimization/ObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModel.cs index 631defae..3d786c18 100644 --- a/src/Numerics/Optimization/ObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModel.cs @@ -36,5 +36,29 @@ namespace MathNet.Numerics.Optimization objective.SetParameters(lowerBound, upperBound, scales, isFixed); return objective; } + + /// + /// Fitting function with a user supplied jacobian for nonlinear least squares regression by the line search algorithm. + /// + public static IObjectiveFunction FittingFunction(Func, double, double> function, Func, double, Vector> derivatives, + Vector observedX, Vector observedY, Vector weight = null) + { + var objective = new FittingObjectiveModel(function, derivatives); + objective.SetObserved(observedX, observedY, weight); + return objective.ToObjectiveFunction(); + } + + /// + /// Fitting function for nonlinear least squares regression by the line search algorithm. + /// The numerical jacobian with accuracy order is used. + /// + public static IObjectiveFunction FittingFunction(Func, double, double> function, + Vector observedX, Vector observedY, Vector weight = null, + int accuracyOrder = 2) + { + var objective = new FittingObjectiveModel(function, null, accuracyOrder: accuracyOrder); + objective.SetObserved(observedX, observedY, weight); + return objective.ToObjectiveFunction(); + } } } diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs index 6b9fba7c..c5558f99 100644 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -1,4 +1,5 @@ using MathNet.Numerics.LinearAlgebra; +using MathNet.Numerics.Optimization.ObjectiveFunctions; using System; using System.Collections.Generic; using System.Linq; @@ -172,7 +173,27 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels public IObjectiveModel CreateNew() { - return new FittingObjectiveModel(userFunction, userDerivatives); + return new FittingObjectiveModel(userFunction, userDerivatives, AccuracyOrder); + } + + public IObjectiveFunction ToObjectiveFunction() + { + Tuple, Matrix> function(Vector point) + { + EvaluateFunction(point); + EvaluateJacobian(point); + + return new Tuple, Matrix>(Residue, -Gradient, Hessian); + } + + LowerBound = null; + UpperBound = null; + Scales = null; + IsFixed = null; + IsBounded = false; + + var objective = new GradientHessianObjectiveFunction(function); + return objective; } /// From 68265cf9cd5a83d13bf52a752817b77dfad6ea2b Mon Sep 17 00:00:00 2001 From: diluculo Date: Wed, 2 Jan 2019 18:22:22 +0900 Subject: [PATCH 08/13] Cleanup NonLinearCurveFittingTests. --- .../NonLinearCurveFittingTests.cs | 201 ++++++++++-------- 1 file changed, 112 insertions(+), 89 deletions(-) diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs index c353b83b..e1fc339f 100644 --- a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -9,6 +9,8 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [TestFixture] public class NonLinearCurveFittingTests { + #region Rosenbrock + // model: Rosenbrock // f(x; a, b) = (1 - a)^2 + 100*(b - a^2)^2 // derivatives: @@ -37,9 +39,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector RosebbrockLowerBound = new DenseVector(new double[] { -5.0, -5.0 }); private Vector RosenbrockUpperBound = new DenseVector(new double[] { 5.0, 5.0 }); + #endregion Rosenbrock + [Test] - public void LMDER_FindMinimum_Rosenbrock_Unconstrained() + public void Rosenbrock_LM_Der() { + // unconstrained var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); var result = solver.FindMinimum(obj, RosenbrockStart1); @@ -48,14 +53,12 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); } - } - [Test] - public void LMDIF_FindMinimum_Rosenbrock_Unconstrained() - { - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder:2); - var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); + // box constrained + obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY, + lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound); + solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); + result = solver.FindMinimum(obj, RosenbrockStart1); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -64,10 +67,10 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void LMDER_FindMinimum_Rosenbrock_BoxConstrained() + public void Rosenbrock_LM_Dif() { - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY, - lowerBound : RosebbrockLowerBound, upperBound : RosenbrockUpperBound); + // unconstrained + var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder:2); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); var result = solver.FindMinimum(obj, RosenbrockStart1); @@ -75,16 +78,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests { AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); } - } - [Test] - public void LMDIF_FindMinimum_Rosenbrock_BoxConstrained() - { - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, + // box constrained + obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound, accuracyOrder: 6); - var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); + solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); + result = solver.FindMinimum(obj, RosenbrockStart1); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -93,54 +93,32 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRDLDER_FindMinimum_Rosenbrock_Unconstrained() + public void Rosenbrock_Bfgs_Dif() { - // DogLeg Minimizer - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); - var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); + var obj = ObjectiveModel.FittingFunction(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); + var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, RosenbrockStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) - { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); - } - - // NewtonCG Minimizer - obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); - var solverNCG = new TrustRegionNewtonCGMinimizer(maximumIterations: 10000); - result = solverNCG.FindMinimum(obj, RosenbrockStart1); - - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 3); } } [Test] - public void TRDLDIF_FindMinimum_Rosenbrock_Unconstrained() + public void Rosenbrock_LBfgs_Dif() { - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); - var solver = new TrustRegionDogLegMinimizer(maximumIterations: 10000); + var obj = ObjectiveModel.FittingFunction(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); + var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, RosenbrockStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 3); } } - [Test] - public void TRNCGDER_FindMinimum_Rosenbrock_Unconstrained() - { - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); - var solver = new TrustRegionNewtonCGMinimizer(maximumIterations: 10000); - var result = solver.FindMinimum(obj, RosenbrockStart1); - - for (int i = 0; i < result.BestFitParameters.Count; i++) - { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 1); - } - } + #region Rat43 // model: Rat43 (https://www.itl.nist.gov/div898/strd/nls/data/ratkowsky3.shtml) // f(x; a, b, c, d) = a / ((1 + exp(b - c * x))^(1 / d)) @@ -172,8 +150,10 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector Rat43Start1 = new DenseVector(new double[] { 100, 10, 1, 1 }); private Vector Rat43Start2 = new DenseVector(new double[] { 700, 5, 0.75, 1.3 }); + #endregion Rat43 + [Test] - public void LMDIF_FindMinimum_Rat43_Unconstrained() + public void Rat43_LM_Dif() { var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); @@ -187,7 +167,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRLMDIF_FindMinimum_Rat43_Unconstrained() + public void Rat43_TRDL_Dif() { var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); @@ -200,6 +180,48 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + [Test] + public void Rat43_TRNCG_Dif() + { + var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var solver = new TrustRegionNewtonCGMinimizer(); + var result = solver.FindMinimum(obj, Rat43Start2); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.BestFitParameters[i], 2); + AssertHelpers.AlmostEqualRelative(Rat43Pstd[i], result.StandardErrors[i], 2); + } + } + + [Test] + public void Rat43_Bfgs_Dif() + { + var obj = ObjectiveModel.FittingFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); + var result = solver.FindMinimum(obj, Rat43Start2); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.MinimizingPoint[i], 2); + } + } + + [Test] + public void Rat43_LBfgs_Dif() + { + var obj = ObjectiveModel.FittingFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); + var result = solver.FindMinimum(obj, Rat43Start2); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.MinimizingPoint[i], 2); + } + } + + #region BoxBod + // model: BoxBod (https://www.itl.nist.gov/div898/strd/nls/data/boxbod.shtml) // f(x; a, b) = a*(1 - exp(-b*x)) // derivatives: @@ -231,24 +253,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector BoxBodUpperBound = new DenseVector(new double[] { 1000.0, 100 }); private Vector BoxBodScales = new DenseVector(new double[] { 100.0, 0.1 }); - [Test] - public void LMDER_FindMinimum_BoxBod_Unconstrained() - { - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); - var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); - - for (int i = 0; i < result.BestFitParameters.Count; i++) - { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); - AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); - } - } + #endregion BoxBod [Test] - public void LMDIF_FindMinimum_BoxBod_Unconstrained() + public void BoxBod_LM_Der() { - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder:6); + // unconstrained + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, BoxBodStart1); @@ -257,18 +268,14 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } - } - [Test] - public void LMDER_FindMinimum_BoxBod_BoxConstrained() - { // lower < parameters < upper // Note that in this case, scales have no effect. - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); - var solver = new LevenbergMarquardtMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -343,11 +350,10 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void LMDIF_FindMinimum_BoxBod_BoxConstrained() + public void BoxBod_LM_Dif() { - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, - lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound, - accuracyOrder: 6); + // unconstrained + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder:6); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, BoxBodStart1); @@ -356,10 +362,23 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } + + // box constrained + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, + lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound, + accuracyOrder: 6); + solver = new LevenbergMarquardtMinimizer(); + result = solver.FindMinimum(obj, BoxBodStart1); + + for (int i = 0; i < result.BestFitParameters.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); + } } [Test] - public void TRDLDIF_FindMinimum_BoxBod_Unconstrained() + public void BoxBod_TRDL_Dif() { var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); @@ -373,7 +392,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRNCGDIF_FindMinimum_BoxBod_Unconstrained() + public void BoxBod_TRNCG_Dif() { var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); @@ -387,7 +406,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void Bfgs_FindMinimum_BoxBod_Unconstrained() + public void BoxBod_Bfgs_Der() { var obj = ObjectiveModel.FittingFunction(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 100); @@ -399,6 +418,8 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + #region Thurber + // model : Thurber (https://www.itl.nist.gov/div898/strd/nls/data/thurber.shtml) // f(x; b1 ... b7) = (b1 + b2*x + b3*x^2 + b4*x^3) / (1 + b5*x + b6*x^2 + b7*x^3) // derivatives: @@ -474,8 +495,10 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector ThurberUpperBound = new DenseVector(new double[] { 1E6, 1E6, 1E6, 1E6, 1E6, 1E6, 1E6 }); private Vector ThurberScales = new DenseVector(new double[7] { 1000, 1000, 400, 40, 0.7, 0.3, 0.03 }); + #endregion Thurber + [Test] - public void LMDER_FindMinimum_Thurber_Unconstrained() + public void Thurber_LM_Der() { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberPrime, ThurberX, ThurberY); var solver = new LevenbergMarquardtMinimizer(); @@ -489,7 +512,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void LMDIF_FindMinimum_Thurber_Unconstrained() + public void Thurber_LM_Dif() { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); @@ -503,7 +526,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRDLDIF_FindMinimum_Thurber_Scaled() + public void Thurber_TRDL_Dif() { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, scales: ThurberScales, @@ -519,7 +542,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void TRNCGDIF_FindMinimum_Thurber_Scaled() + public void Thurber_TRNCG_Dif() { var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, scales: ThurberScales, @@ -535,7 +558,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void Bfgs_FindMinimum_Thurber_Unconstrained() + public void Thurber_Bfgs_Dif() { var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); @@ -548,7 +571,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void BfgsB_FindMinimum_Thurber() + public void Thurber_BfgsB_Dif() { var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new BfgsBMinimizer(1e-10, 1e-10, 1e-10, 1000); @@ -561,7 +584,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } [Test] - public void LBfgs_FindMinimum_Thurber() + public void Thurber_LBfgs_Dif() { var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); From 478d6a38a4d30a80ef2040342b6bebe88ad1a57a Mon Sep 17 00:00:00 2001 From: diluculo Date: Wed, 2 Jan 2019 19:09:49 +0900 Subject: [PATCH 09/13] Renamed ModelMinimizationResult to NonlinearMinimizationResult. --- .../Optimization/LevenbergMarquardtMinimizer.cs | 12 ++++++------ ...ationResult.cs => NonlinearMinimizationResult.cs} | 4 ++-- .../Optimization/Subproblems/DogLegSubproblem.cs | 1 - .../Optimization/Subproblems/NewtonCGSubproblem.cs | 1 - .../Optimization/TrustRegionMinimizerBase.cs | 12 ++++++------ 5 files changed, 14 insertions(+), 16 deletions(-) rename src/Numerics/Optimization/{ModelMinimizationResult.cs => NonlinearMinimizationResult.cs} (90%) diff --git a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs index 2b54d194..ddae4d72 100644 --- a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs +++ b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs @@ -44,7 +44,7 @@ namespace MathNet.Numerics.Optimization MaximumIterations = maximumIterations; } - public ModelMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) { if (objective == null) throw new ArgumentNullException("objective"); @@ -54,7 +54,7 @@ namespace MathNet.Numerics.Optimization return Minimum(objective, initialGuess, InitialMu, FunctionTolerance, GradientTolerance, StepTolerance, MaximumIterations); } - public ModelMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) { if (objective == null) throw new ArgumentNullException("objective"); @@ -75,7 +75,7 @@ namespace MathNet.Numerics.Optimization /// The stopping threshold for L2 norm of the residuals. /// The max iterations. /// The result of the Levenberg-Marquardt minimization - public static ModelMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) { // Non-linear least square fitting by the Levenberg-Marduardt algorithm. // @@ -141,7 +141,7 @@ namespace MathNet.Numerics.Optimization if (double.IsNaN(RSS)) { exitCondition = ExitCondition.InvalidValues; - return new ModelMinimizationResult(objective, -1, exitCondition); + return new NonlinearMinimizationResult(objective, -1, exitCondition); } // When only function evaluation is needed, set maximumIterations to zero, @@ -171,7 +171,7 @@ namespace MathNet.Numerics.Optimization if (exitCondition != ExitCondition.None) { objective.EvaluateCovariance(P); - return new ModelMinimizationResult(objective, -1, exitCondition); + return new NonlinearMinimizationResult(objective, -1, exitCondition); } double mu = initialMu * diagonalOfHessian.Max(); // μ @@ -261,7 +261,7 @@ namespace MathNet.Numerics.Optimization // finalize objective.EvaluateCovariance(P); - return new ModelMinimizationResult(objective, iterations, exitCondition); + return new NonlinearMinimizationResult(objective, iterations, exitCondition); } } } diff --git a/src/Numerics/Optimization/ModelMinimizationResult.cs b/src/Numerics/Optimization/NonlinearMinimizationResult.cs similarity index 90% rename from src/Numerics/Optimization/ModelMinimizationResult.cs rename to src/Numerics/Optimization/NonlinearMinimizationResult.cs index baaf3710..a88569ba 100644 --- a/src/Numerics/Optimization/ModelMinimizationResult.cs +++ b/src/Numerics/Optimization/NonlinearMinimizationResult.cs @@ -6,7 +6,7 @@ using System.Text; namespace MathNet.Numerics.Optimization { - public class ModelMinimizationResult + public class NonlinearMinimizationResult { public IObjectiveModel ModelInfoAtMinimum { get; private set; } @@ -42,7 +42,7 @@ namespace MathNet.Numerics.Optimization public int Iterations { get; private set; } public ExitCondition ReasonForExit { get; private set; } - public ModelMinimizationResult(IObjectiveModel modelInfo, int iterations, ExitCondition reasonForExit) + public NonlinearMinimizationResult(IObjectiveModel modelInfo, int iterations, ExitCondition reasonForExit) { ModelInfoAtMinimum = modelInfo; Iterations = iterations; diff --git a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs index 8d41288d..83671ce7 100644 --- a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs @@ -10,7 +10,6 @@ namespace MathNet.Numerics.Optimization.Subproblems public void Solve(IObjectiveModel objective, double delta) { - var Jacobian = objective.Jacobian; var Gradient = objective.Gradient; var Hessian = objective.Hessian; diff --git a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs index ba6f9a6c..04c98214 100644 --- a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs @@ -11,7 +11,6 @@ namespace MathNet.Numerics.Optimization.Subproblems public void Solve(IObjectiveModel objective, double delta) { - var Jacobian = objective.Jacobian; var Gradient = objective.Gradient; var Hessian = objective.Hessian; diff --git a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs index f4125e8f..50711a69 100644 --- a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs +++ b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs @@ -46,7 +46,7 @@ namespace MathNet.Numerics.Optimization MaximumIterations = maximumIterations; } - public ModelMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) { if (objective == null) throw new ArgumentNullException("objective"); @@ -56,7 +56,7 @@ namespace MathNet.Numerics.Optimization return Minimum(objective, initialGuess, Subproblem, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); } - public ModelMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) { if (objective == null) throw new ArgumentNullException("objective"); @@ -78,7 +78,7 @@ namespace MathNet.Numerics.Optimization /// The stopping threshold for trust region radius /// The max iterations. /// - public static ModelMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, ITrustRegionSubproblem subproblem, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) + public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, ITrustRegionSubproblem subproblem, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) { // Non-linear least square fitting by the trust-region algorithm. // @@ -144,7 +144,7 @@ namespace MathNet.Numerics.Optimization if (double.IsNaN(RSS)) { exitCondition = ExitCondition.InvalidValues; - return new ModelMinimizationResult(objective, -1, exitCondition); + return new NonlinearMinimizationResult(objective, -1, exitCondition); } // When only function evaluation is needed, set maximumIterations to zero, @@ -174,7 +174,7 @@ namespace MathNet.Numerics.Optimization { // finalize objective.EvaluateCovariance(P); - return new ModelMinimizationResult(objective, -1, exitCondition); + return new NonlinearMinimizationResult(objective, -1, exitCondition); } // initialize trust-region radius, Δ @@ -256,7 +256,7 @@ namespace MathNet.Numerics.Optimization // finalize objective.EvaluateCovariance(P); - return new ModelMinimizationResult(objective, iterations, exitCondition); + return new NonlinearMinimizationResult(objective, iterations, exitCondition); } } } From 99abb36a8865aa02b20a8a506ef476ba880e6034 Mon Sep 17 00:00:00 2001 From: diluculo Date: Wed, 2 Jan 2019 20:13:32 +0900 Subject: [PATCH 10/13] Add Correlation matrix to ObjectiveModel. --- src/Numerics/Optimization/IObjectiveModel.cs | 15 +++++++++++++++ .../ObjectiveModels/FittingObjectiveModel.cs | 16 +++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/Numerics/Optimization/IObjectiveModel.cs b/src/Numerics/Optimization/IObjectiveModel.cs index 2409dffe..ff117091 100644 --- a/src/Numerics/Optimization/IObjectiveModel.cs +++ b/src/Numerics/Optimization/IObjectiveModel.cs @@ -6,6 +6,16 @@ namespace MathNet.Numerics.Optimization { IObjectiveModel CreateNew(); + /// + /// Get the y-values of the observations. + /// + Vector ObservedY { get; } + + /// + /// Get the values of the weights for the observations. + /// + Matrix Weights { get; } + /// /// Get the y-values of the fitted model that correspond to the independent values. /// @@ -38,6 +48,11 @@ namespace MathNet.Numerics.Optimization /// Matrix Covariance { get; } + /// + /// Get the correlation matrix. + /// + Matrix Correlation { get; } + /// /// Get the number of calls to function. /// diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs index c5558f99..f1204189 100644 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -123,6 +123,11 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels /// public Matrix Covariance { get; private set; } + /// + /// Get the correlation matrix. + /// + public Matrix Correlation { get; private set; } + /// /// Get the number of calls to function. /// @@ -484,18 +489,23 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels EvaluateFunction(Pext); EvaluateJacobian(Pext); + // restore isBounded + this.IsBounded = (LowerBound != null || UpperBound != null); + if (Hessian == null || Residuals == null || DegreeOfFreedom < 1) { Covariance = null; + Correlation = null; return; } var covariance = Hessian.PseudoInverse() * Residuals.DotProduct(Residuals) / DegreeOfFreedom; - Covariance = covariance; - // restore isBounded - this.IsBounded = (LowerBound != null || UpperBound != null); + var correlation = covariance.Clone(); + var d = correlation.Diagonal().PointwiseSqrt(); + var dd = d.OuterProduct(d); + Correlation = correlation.PointwiseDivide(dd); return; } From 975cd212520d1540607275d2d9adb2366ae10876 Mon Sep 17 00:00:00 2001 From: diluculo Date: Thu, 3 Jan 2019 10:13:21 +0900 Subject: [PATCH 11/13] Changed the sign of the gradient of the FittingObjectiveModel to match the scheme of the existing Minimizers. --- .../Optimization/LevenbergMarquardtMinimizer.cs | 14 +++++++------- .../ObjectiveModels/FittingObjectiveModel.cs | 8 ++++---- .../Optimization/Subproblems/DogLegSubproblem.cs | 4 ++-- .../Optimization/Subproblems/NewtonCGSubproblem.cs | 5 +++-- .../Optimization/TrustRegionMinimizerBase.cs | 6 +++--- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs index ddae4d72..ab6bb421 100644 --- a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs +++ b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs @@ -92,14 +92,14 @@ namespace MathNet.Numerics.Optimization // Residuals, R = L(y - f(x; p)) // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) // Jacobian J = df(x; p)/dp - // Gradient g = J'W(y − f(x; p)) = J'LR + // Gradient g = -J'W(y − f(x; p)) = -J'LR // Approximated Hessian H = J'WJ // // The Levenberg-Marquardt algorithm is summarized as follows: - // initially let μ = τ * max(diag(J'WJ)). + // initially let μ = τ * max(diag(H)). // repeat - // solve linear equations: (J'WJ + μI)ΔP = J'R - // let ρ = (||R||^2 - ||Rnew||^2) / (Δp'(μΔp + J'R)). + // solve linear equations: (H + μI)ΔP = -g + // let ρ = (||R||^2 - ||Rnew||^2) / (Δp'(μΔp - g)). // if ρ > ε, P = P + ΔP; μ = μ * max(1/3, 1 - (2ρ - 1)^3); ν = 2; // otherwise μ = μ*ν; ν = 2*ν; // @@ -186,7 +186,7 @@ namespace MathNet.Numerics.Optimization Hessian.SetDiagonal(Hessian.Diagonal() + mu); // hessian[i, i] = hessian[i, i] + mu; // solve normal equations - Pstep = Hessian.Solve(Gradient); + Pstep = Hessian.Solve(-Gradient); // if ||ΔP|| <= xTol * (||P|| + xTol), found and stop if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.DotProduct(P))) @@ -207,8 +207,8 @@ namespace MathNet.Numerics.Optimization } // calculate the ratio of the actual to the predicted reduction. - // ρ = (RSS - RSSnew) / (Δp'(μΔp + g)) - var predictedReduction = Pstep.DotProduct(mu * Pstep + Gradient); + // ρ = (RSS - RSSnew) / (Δp'(μΔp - g)) + var predictedReduction = Pstep.DotProduct(mu * Pstep - Gradient); var rho = (predictedReduction != 0) ? (RSS - RSSnew) / predictedReduction : 0; diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs index f1204189..22401f63 100644 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -188,7 +188,7 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels EvaluateFunction(point); EvaluateJacobian(point); - return new Tuple, Matrix>(Residue, -Gradient, Hessian); + return new Tuple, Matrix>(Residue, Gradient, Hessian); } LowerBound = null; @@ -465,10 +465,10 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels } } - // Gradient, g = J'W(y − f(x; p)) = J'L(L'E) = J'LR + // Gradient, g = -J'W(y − f(x; p)) = -J'L(L'E) = -J'LR Gradient = (Weights == null) - ? Jacobian.Transpose() * (ObservedY - Values) - : Jacobian.Transpose() * Weights * (ObservedY - Values); + ? -Jacobian.Transpose() * (ObservedY - Values) + : -Jacobian.Transpose() * Weights * (ObservedY - Values); // approximated Hessian, H = J'WJ + ∑LRiHi ~ J'WJ near the minimum Hessian = (Weights == null) diff --git a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs index 83671ce7..eb62acef 100644 --- a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs @@ -15,12 +15,12 @@ namespace MathNet.Numerics.Optimization.Subproblems // newton point // the Gauss–Newton step by solving the normal equations - var Pgn = Hessian.PseudoInverse() * Gradient; // Hessian.Solve(Gradient) fails so many times... + var Pgn = -Hessian.PseudoInverse() * Gradient; // Hessian.Solve(Gradient) fails so many times... // cauchy point // steepest descent direction is given by var alpha = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); - var Psd = alpha * Gradient; + var Psd = -alpha * Gradient; // update step and prectted reduction if (Pgn.L2Norm() <= delta) diff --git a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs index 04c98214..ec26b34a 100644 --- a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs @@ -15,11 +15,12 @@ namespace MathNet.Numerics.Optimization.Subproblems var Hessian = objective.Hessian; // define tolerance - var tolerance = Math.Min(0.5, Math.Sqrt(Gradient.L2Norm())) * Gradient.L2Norm(); + var gnorm = Gradient.L2Norm(); + var tolerance = Math.Min(0.5, Math.Sqrt(gnorm)) * gnorm; // initialize internal variables var z = Vector.Build.Dense(Hessian.RowCount); - var r = -Gradient; + var r = Gradient; var d = -r; while (true) diff --git a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs index 50711a69..22de97d8 100644 --- a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs +++ b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs @@ -93,7 +93,7 @@ namespace MathNet.Numerics.Optimization // Residuals, R = L(y - f(x; p)) // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) // Jacobian J = df(x; p)/dp - // Gradient g = J'W(y − f(x; p)) = J'LR + // Gradient g = -J'W(y − f(x; p)) = -J'LR // Approximated Hessian H = J'WJ // // The trust region algorithm is summarized as follows: @@ -190,8 +190,8 @@ namespace MathNet.Numerics.Optimization subproblem.Solve(objective, delta); var Pstep = subproblem.Pstep; var hitBoundary = subproblem.HitBoundary; - // predicted reduction = L(0) - L(Δp) = Δp'g - 1/2 * Δp'HΔp - var predictedReduction = objective.Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(objective.Hessian * Pstep); + // predicted reduction = L(0) - L(Δp) = -Δp'g - 1/2 * Δp'HΔp + var predictedReduction = -objective.Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(objective.Hessian * Pstep); if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) { From 22d9041d59d48c667d0fb605ecbec9ad34e7e392 Mon Sep 17 00:00:00 2001 From: diluculo Date: Thu, 3 Jan 2019 21:52:52 +0900 Subject: [PATCH 12/13] Reworked ObjectiveModel. --- .../NonLinearCurveFittingTests.cs | 64 +-- src/Numerics/Optimization/IObjectiveModel.cs | 36 +- .../LevenbergMarquardtMinimizer.cs | 45 +- .../NonlinearMinimizationResult.cs | 54 +- src/Numerics/Optimization/ObjectiveModel.cs | 12 +- .../ObjectiveModels/FittingObjectiveModel.cs | 500 ++++++++---------- .../Subproblems/DogLegSubproblem.cs | 6 +- .../Subproblems/NewtonCGSubproblem.cs | 2 +- .../Optimization/TrustRegionMinimizerBase.cs | 64 ++- 9 files changed, 376 insertions(+), 407 deletions(-) diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs index e1fc339f..faae726c 100644 --- a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -55,10 +55,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } // box constrained - obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY, - lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound); + obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - result = solver.FindMinimum(obj, RosenbrockStart1); + result = solver.FindMinimum(obj, RosenbrockStart1, lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -80,11 +79,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } // box constrained - obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, - lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound, - accuracyOrder: 6); + obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); - result = solver.FindMinimum(obj, RosenbrockStart1); + result = solver.FindMinimum(obj, RosenbrockStart1, lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -272,10 +269,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // lower < parameters < upper // Note that in this case, scales have no effect. - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, - lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -285,10 +281,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // lower < parameters, no scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, - lowerBound: BoxBodLowerBound); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -298,10 +293,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // lower < parameters, scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, - lowerBound: BoxBodLowerBound, scales: BoxBodScales); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound, scales: BoxBodScales); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -311,10 +305,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // parameters < upper, no scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, - upperBound: BoxBodUpperBound); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, upperBound: BoxBodUpperBound); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -324,10 +317,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // parameters < upper, scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, - upperBound: BoxBodUpperBound, scales: BoxBodScales); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, upperBound: BoxBodUpperBound, scales: BoxBodScales); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -337,10 +329,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // only scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY, - scales: BoxBodScales); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, scales: BoxBodScales); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -364,11 +355,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } // box constrained - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, - lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound, - accuracyOrder: 6); + obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); solver = new LevenbergMarquardtMinimizer(); - result = solver.FindMinimum(obj, BoxBodStart1); + result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -394,9 +383,10 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void BoxBod_TRNCG_Dif() { - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); + var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + //var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); - var result = solver.FindMinimum(obj, BoxBodStart1); + var result = solver.FindMinimum(obj, BoxBodStart2); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -528,11 +518,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_TRDL_Dif() { - var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, - scales: ThurberScales, - accuracyOrder: 6); + var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); - var result = solver.FindMinimum(obj, ThurberStart); + var result = solver.FindMinimum(obj, ThurberStart, scales: ThurberScales); for (int i = 0; i < result.BestFitParameters.Count; i++) { @@ -544,11 +532,9 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_TRNCG_Dif() { - var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, - scales: ThurberScales, - accuracyOrder: 6); + var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); - var result = solver.FindMinimum(obj, ThurberStart); + var result = solver.FindMinimum(obj, ThurberStart, scales: ThurberScales); for (int i = 0; i < result.BestFitParameters.Count; i++) { diff --git a/src/Numerics/Optimization/IObjectiveModel.cs b/src/Numerics/Optimization/IObjectiveModel.cs index ff117091..b106a347 100644 --- a/src/Numerics/Optimization/IObjectiveModel.cs +++ b/src/Numerics/Optimization/IObjectiveModel.cs @@ -1,4 +1,5 @@ using MathNet.Numerics.LinearAlgebra; +using System.Collections.Generic; namespace MathNet.Numerics.Optimization { @@ -19,44 +20,33 @@ namespace MathNet.Numerics.Optimization /// /// Get the y-values of the fitted model that correspond to the independent values. /// - Vector Values { get; } + Vector ModelValues { get; } /// /// Get the values of the parameters. /// - Vector Parameters { get; } + Vector Point { get; } /// /// Get the residual sum of squares. /// - double Residue { get; } + double Value { get; } - /// - /// Get the Jacobian matrix, J(x; p) = df(x; p)/dp. - /// - Matrix Jacobian { get; } /// /// Get the Gradient vector. G = J'(y - f(x; p)) /// Vector Gradient { get; } + /// /// Get the approximated Hessian matrix. H = J'J /// Matrix Hessian { get; } - /// - /// Get the covariance matrix. - /// - Matrix Covariance { get; } - - /// - /// Get the correlation matrix. - /// - Matrix Correlation { get; } /// /// Get the number of calls to function. /// int FunctionEvaluations { get; set; } + /// /// Get the number of calls to jacobian. /// @@ -67,17 +57,17 @@ namespace MathNet.Numerics.Optimization /// int DegreeOfFreedom { get; } - /// - /// Get whether or not the analytical jacobian is supported. - /// - bool IsJacobianSupported { get; } + bool IsGradientSupported { get; } + bool IsHessianSupported { get; } + + bool IsFinished { get; set; } } public interface IObjectiveModel : IObjectiveModelEvaluation { - void EvaluateFunction(Vector parameters); - void EvaluateJacobian(Vector parameters); - void EvaluateCovariance(Vector parameters); + void SetParameters(Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null); + + void EvaluateAt(Vector parameters); /// Create a new independent copy of this objective function, evaluated at the same point. IObjectiveModel Fork(); diff --git a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs index ab6bb421..c39b0119 100644 --- a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs +++ b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs @@ -1,5 +1,6 @@ using MathNet.Numerics.LinearAlgebra; using System; +using System.Collections.Generic; using System.Linq; namespace MathNet.Numerics.Optimization @@ -44,24 +45,31 @@ namespace MathNet.Numerics.Optimization MaximumIterations = maximumIterations; } - public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess, + Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) { if (objective == null) throw new ArgumentNullException("objective"); if (initialGuess == null) throw new ArgumentNullException("initialGuess"); - return Minimum(objective, initialGuess, InitialMu, FunctionTolerance, GradientTolerance, StepTolerance, MaximumIterations); + return Minimum(objective, initialGuess, lowerBound, upperBound, scales, isFixed, InitialMu, FunctionTolerance, GradientTolerance, StepTolerance, MaximumIterations); } - public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess, + double[] lowerBound = null, double[] upperBound = null, double[] scales = null, bool[] isFixed = null) { if (objective == null) throw new ArgumentNullException("objective"); if (initialGuess == null) throw new ArgumentNullException("initialGuess"); - return Minimum(objective, CreateVector.DenseOfArray(initialGuess), InitialMu, GradientTolerance, StepTolerance, FunctionTolerance, MaximumIterations); + var lb = (lowerBound == null) ? null : CreateVector.Dense(lowerBound); + var ub = (upperBound == null) ? null : CreateVector.Dense(upperBound); + var sc = (scales == null) ? null : CreateVector.Dense(scales); + var fx = (isFixed == null) ? null : isFixed.ToList(); + + return Minimum(objective, CreateVector.DenseOfArray(initialGuess), lb, ub, sc, fx, InitialMu, GradientTolerance, StepTolerance, FunctionTolerance, MaximumIterations); } /// @@ -75,7 +83,9 @@ namespace MathNet.Numerics.Optimization /// The stopping threshold for L2 norm of the residuals. /// The max iterations. /// The result of the Levenberg-Marquardt minimization - public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, + Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null, + double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) { // Non-linear least square fitting by the Levenberg-Marduardt algorithm. // @@ -118,23 +128,24 @@ namespace MathNet.Numerics.Optimization if (initialGuess == null) throw new ArgumentNullException("initialGuess"); + objective.SetParameters(initialGuess, lowerBound, upperBound, scales, isFixed); + ExitCondition exitCondition = ExitCondition.None; // Initialize objective objective.FunctionEvaluations = 0; objective.JacobianEvaluations = 0; + objective.IsFinished = false; // First, calculate function values and setup variables - objective.EvaluateFunction(initialGuess); - var P = objective.Parameters; // current parameters + objective.EvaluateAt(initialGuess); + var P = objective.Point; // current parameters var Pstep = Vector.Build.Dense(P.Count); // the change of parameters - var RSS = objective.Residue; // Residual Sum of Squares = R'R + var RSS = objective.Value; // Residual Sum of Squares = R'R if (maximumIterations < 0) { - maximumIterations = (objective.IsJacobianSupported) - ? 100 * (initialGuess.Count + 1) - : 200 * (initialGuess.Count + 1); + maximumIterations = 200 * (initialGuess.Count + 1); } // if RSS == NaN, stop @@ -157,9 +168,8 @@ namespace MathNet.Numerics.Optimization } // Evaluate gradient and Hessian - objective.EvaluateJacobian(P); var Gradient = objective.Gradient; - var Hessian = objective.Hessian; + var Hessian = objective.Hessian; var diagonalOfHessian = Hessian.Diagonal(); // diag(H) // if ||g||oo <= gtol, found and stop @@ -170,7 +180,6 @@ namespace MathNet.Numerics.Optimization if (exitCondition != ExitCondition.None) { - objective.EvaluateCovariance(P); return new NonlinearMinimizationResult(objective, -1, exitCondition); } @@ -197,8 +206,8 @@ namespace MathNet.Numerics.Optimization var Pnew = P + Pstep; // new parameters to test - objective.EvaluateFunction(Pnew); - var RSSnew = objective.Residue; + objective.EvaluateAt(Pnew); + var RSSnew = objective.Value; if (double.IsNaN(RSSnew)) { @@ -220,7 +229,6 @@ namespace MathNet.Numerics.Optimization RSS = RSSnew; // update gradient and Hessian - objective.EvaluateJacobian(P); Gradient = objective.Gradient; Hessian = objective.Hessian; diagonalOfHessian = Hessian.Diagonal(); @@ -258,9 +266,6 @@ namespace MathNet.Numerics.Optimization exitCondition = ExitCondition.ExceedIterations; } - // finalize - objective.EvaluateCovariance(P); - return new NonlinearMinimizationResult(objective, iterations, exitCondition); } } diff --git a/src/Numerics/Optimization/NonlinearMinimizationResult.cs b/src/Numerics/Optimization/NonlinearMinimizationResult.cs index a88569ba..12381bbf 100644 --- a/src/Numerics/Optimization/NonlinearMinimizationResult.cs +++ b/src/Numerics/Optimization/NonlinearMinimizationResult.cs @@ -13,31 +13,25 @@ namespace MathNet.Numerics.Optimization /// /// Returns the best fit parameters. /// - public Vector BestFitParameters { get { return ModelInfoAtMinimum.Parameters; } } + public Vector BestFitParameters { get { return ModelInfoAtMinimum.Point; } } /// /// Returns the standard errors of the corresponding parameters /// - public Vector StandardErrors - { - get - { - if (ModelInfoAtMinimum.Covariance == null) - return null; - return ModelInfoAtMinimum.Covariance.Diagonal().PointwiseSqrt(); - } - } + public Vector StandardErrors { get; private set; } /// /// Returns the y-values of the fitted model that correspond to the independent values. /// - public Vector BestFitValues { get { return ModelInfoAtMinimum.Values; } } + public Vector BestFitValues { get { return ModelInfoAtMinimum.ModelValues; } } /// /// Returns the residual sum of squares. /// - public double Residue { get { return ModelInfoAtMinimum.Residue; } } - public double DegreeOfFreedom { get { return ModelInfoAtMinimum.DegreeOfFreedom; } } + public double Residue { get { return ModelInfoAtMinimum.Value; } } + public double DegreeOfFreedom { get { return ModelInfoAtMinimum.DegreeOfFreedom; } } + public Matrix Covariance { get; private set; } + public Matrix Correlation { get; private set; } public int Iterations { get; private set; } public ExitCondition ReasonForExit { get; private set; } @@ -47,6 +41,40 @@ namespace MathNet.Numerics.Optimization ModelInfoAtMinimum = modelInfo; Iterations = iterations; ReasonForExit = reasonForExit; + + AnalyzeResult(modelInfo); + } + + private void AnalyzeResult(IObjectiveModel objective) + { + objective.IsFinished = true; + objective.EvaluateAt(objective.Point); + + var Hessian = objective.Hessian; + if (Hessian == null || DegreeOfFreedom < 1) + { + Covariance = null; + Correlation = null; + StandardErrors = null; + return; + } + + Covariance = Hessian.PseudoInverse() * objective.Value / DegreeOfFreedom; + + if (Covariance != null) + { + StandardErrors = Covariance.Diagonal().PointwiseSqrt(); + + var correlation = Covariance.Clone(); + var d = correlation.Diagonal().PointwiseSqrt(); + var dd = d.OuterProduct(d); + Correlation = correlation.PointwiseDivide(dd); + } + else + { + StandardErrors = null; + Correlation = null; + } } } } diff --git a/src/Numerics/Optimization/ObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModel.cs index 3d786c18..77cae337 100644 --- a/src/Numerics/Optimization/ObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModel.cs @@ -1,9 +1,6 @@ using MathNet.Numerics.LinearAlgebra; using MathNet.Numerics.Optimization.ObjectiveModels; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; namespace MathNet.Numerics.Optimization { @@ -13,27 +10,22 @@ namespace MathNet.Numerics.Optimization /// Fitting model with a user supplied jacobian for non-linear least squares regression. /// public static IObjectiveModel FittingModel(Func, double, double> function, Func, double, Vector> derivatives, - Vector observedX, Vector observedY, Vector weight = null, - Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) + Vector observedX, Vector observedY, Vector weight = null) { var objective = new FittingObjectiveModel(function, derivatives); objective.SetObserved(observedX, observedY, weight); - objective.SetParameters(lowerBound, upperBound, scales, isFixed); return objective; } /// /// Fitting model for non-linear least squares regression. - /// The numerical jacobian with accuracy order is used. /// public static IObjectiveModel FittingModel(Func, double, double> function, Vector observedX, Vector observedY, Vector weight = null, - Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null, int accuracyOrder = 2) { - var objective = new FittingObjectiveModel(function, null, accuracyOrder: accuracyOrder); + var objective = new FittingObjectiveModel(function, accuracyOrder: accuracyOrder); objective.SetObserved(observedX, observedY, weight); - objective.SetParameters(lowerBound, upperBound, scales, isFixed); return objective; } diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs index 22401f63..eb30f357 100644 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs @@ -8,10 +8,30 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { internal class FittingObjectiveModel : IObjectiveModel { + #region Private Variables + readonly Func, double, double> userFunction; // (p, x) => f(x; p) readonly Func, double, Vector> userDerivatives; // (p, x) => df(x; p)/dp + readonly int accuracyOrder; // the desired accuracy order to evaluate the jacobian by numerical approximaiton. + + Vector coefficients; + Vector Pint; // internal(unbounded) coefficients + public Vector Pext; // external(bounded) coefficients + + bool hasFunctionValue; + double functionValue; // the residual sum of squares. Residuals * Residuals + Vector residuals; // the error values + + bool hasJacobianValue; + Matrix jacobianValue; // the Jacobian matrix. + Vector gradientValue; // the Gradient vector. + Matrix hessianValue; // the Hessian matrix. + + bool isBounded; + + #endregion Private Variables - #region Public Variables + #region Public Variables - Observed Data /// /// Set or get the values of the independent variable. @@ -25,178 +45,183 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels /// /// Set or get the values of the weights for the observations. - /// inverse of the standard measurement errors - /// If null, unity weighting is used. /// public Matrix Weights { get; private set; } - // W = LL' - private Vector L; + private Vector L; // Weights = LL' /// - /// Set or get the values of the parameters. + /// Get the number of observations. /// - public Vector Parameters { get; private set; } + public int NumberOfObservations { get { return (ObservedY == null) ? 0 : ObservedY.Count; } } - /// - /// Set or get the values of the parameters. - /// - public List IsFixed { get; set; } + #endregion Public Variables - Observed Data - /// - /// Set or get the values of the parameters. - /// - public Vector LowerBound { get; set; } + #region Public Variables - Bounds of Parameter /// - /// Set or get the values of the parameters. + /// Get the values of the parameters. /// - public Vector UpperBound { get; set; } + public List IsFixed { get; private set; } /// - /// Set or get the scale factor of the parameters. + /// Get the values of the parameters. /// - public Vector Scales { get; set; } + public Vector LowerBound { get; private set; } /// - /// Set of get whether or not the parameters are bounded. + /// Get the values of the parameters. /// - public bool IsBounded { get; set; } + public Vector UpperBound { get; private set; } /// - /// Get the y-values of the fitted model that correspond to the independent values. + /// Get the scale factor of the parameters. /// - public Vector Values { get; private set; } + public Vector Scales { get; private set; } /// - /// Get the error values, R(x; p) = L * (y - f(x; p)) where L = sqrt(W) + /// Get the number of unknown parameters. /// - private Vector Residuals; + public int NumberOfParameters { get { return (Point == null) ? 0 : Point.Count; } } - /// - /// Get the residual sum of squares, R.DotProduct(R) - /// - public double Residue { get; private set; } + #endregion Public Variables - Bounds of Parameter + + #region Public Variables - Others /// - /// Get the Jacobian matrix of x and p, J(x; p). + /// Get the number of calls to function. /// - public Matrix Jacobian { get; private set; } - + public int FunctionEvaluations { get; set; } /// - /// Get the Gradient vector of x and p, J'WR + /// Get the number of calls to jacobian. /// - public Vector Gradient { get; private set; } + public int JacobianEvaluations { get; set; } - /// - /// Get the Hessian matrix of x and p, J'WJ - /// - public Matrix Hessian { get; private set; } + #endregion Public Variables - Others + + public FittingObjectiveModel(Func, double, double>function, Func, double, Vector> derivatives = null, int accuracyOrder = 2) + { + this.userFunction = function; + this.userDerivatives = derivatives; + this.accuracyOrder = Math.Min(6, Math.Max(1, accuracyOrder)); + + IsFinished = false; + } + + public IObjectiveModel Fork() + { + return new FittingObjectiveModel(userFunction, userDerivatives, accuracyOrder) + { + ObservedX = ObservedX, + ObservedY = ObservedY, + Weights = Weights, + + coefficients = coefficients, + Pint = Pint, + Pext = Pext, + + hasFunctionValue = hasFunctionValue, + functionValue = functionValue, + + hasJacobianValue = hasJacobianValue, + jacobianValue = jacobianValue, + gradientValue = gradientValue, + hessianValue = hessianValue + }; + } + + public IObjectiveModel CreateNew() + { + return new FittingObjectiveModel(userFunction, userDerivatives, accuracyOrder); + } /// - /// Get the number of observations. + /// Set or get the values of the parameters. /// - public int NumberOfObservations { get { return (ObservedY == null) ? 0 : ObservedY.Count; } } + public Vector Point { get { return coefficients; } } /// - /// Get the number of unknown parameters. + /// Get the y-values of the fitted model that correspond to the independent values. /// - public int NumberOfParameters { get { return (Parameters == null) ? 0 : Parameters.Count; } } + public Vector ModelValues { get; private set; } /// - /// Get the degree of freedom + /// Get the residual sum of squares. /// - public int DegreeOfFreedom + public double Value { get { - var dof = NumberOfObservations - NumberOfParameters; - if (IsFixed != null) + if (!hasFunctionValue) { - dof = dof + IsFixed.Count(p => p == true); + EvaluateFunction(); + hasFunctionValue = true; } - return dof; + return functionValue; } } /// - /// Get the covariance matrix. - /// - public Matrix Covariance { get; private set; } - - /// - /// Get the correlation matrix. - /// - public Matrix Correlation { get; private set; } - - /// - /// Get the number of calls to function. - /// - public int FunctionEvaluations { get; set; } - /// - /// Get the number of calls to jacobian. - /// - public int JacobianEvaluations { get; set; } - - /// - /// Set or get the desired accuracy order of the numerical jacobian. + /// Get the Gradient vector of x and p. /// - public int AccuracyOrder { get; set; } + public Vector Gradient + { + get + { + if (!hasJacobianValue) + { + EvaluateJacobian(); + hasJacobianValue = true; + } + return gradientValue; + } + } /// - /// Get whether or not the analytical jacobian is supported. + /// Get the Hessian matrix of x and p, J'WJ /// - public bool IsJacobianSupported { get { return userDerivatives != null; } } - - #endregion Public Variables - - public FittingObjectiveModel(Func, double, double>function, Func, double, Vector> derivatives, int accuracyOrder = 2) + public Matrix Hessian { - userFunction = function; - userDerivatives = derivatives; - AccuracyOrder = Math.Min(6, Math.Max(1, accuracyOrder)); + get + { + if (!hasJacobianValue) + { + EvaluateJacobian(); + hasJacobianValue = true; + } + return hessianValue; + } } - public IObjectiveModel Fork() + /// + /// Get the degree of freedom + /// + public int DegreeOfFreedom { - return new FittingObjectiveModel(userFunction, userDerivatives, AccuracyOrder) + get { - ObservedX = ObservedX, - ObservedY = ObservedY, - Weights = Weights, - - Parameters = Parameters, - LowerBound = LowerBound, - UpperBound = UpperBound, - IsFixed = IsFixed, - Scales = Scales, - IsBounded = IsBounded, - - Residue = Residue, - Jacobian = Jacobian - }; + var df = NumberOfObservations - NumberOfParameters; + if (IsFixed != null) + { + df = df + IsFixed.Count(p => p == true); + } + return df; + } } - public IObjectiveModel CreateNew() - { - return new FittingObjectiveModel(userFunction, userDerivatives, AccuracyOrder); - } + public bool IsGradientSupported { get { return true; } } + public bool IsHessianSupported { get { return true; } } + + public bool IsFinished { get; set; } public IObjectiveFunction ToObjectiveFunction() { Tuple, Matrix> function(Vector point) { - EvaluateFunction(point); - EvaluateJacobian(point); + EvaluateAt(point); - return new Tuple, Matrix>(Residue, Gradient, Hessian); + return new Tuple, Matrix>(Value, Gradient, Hessian); } - LowerBound = null; - UpperBound = null; - Scales = null; - IsFixed = null; - IsBounded = false; - var objective = new GradientHessianObjectiveFunction(function); return objective; } @@ -244,50 +269,37 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels } /// - /// Set observed data to fit. - /// - public void SetObserved(double[] observedX, double[] observedY, double[] weights = null) - { - if (observedX == null || observedY == null) - { - throw new ArgumentNullException("The data set can't be null."); - } - if (observedX.Length != observedY.Length) - { - throw new ArgumentException("The observed x data can't have different from observed y data."); - } - - var wVector = (weights == null) - ? null - : Vector.Build.DenseOfArray(weights); - SetObserved(Vector.Build.DenseOfArray(observedX), Vector.Build.DenseOfArray(observedY), wVector); - } - - /// - /// Set parameters. - /// - /// If bounded, the paramneters will be projected to unconstrained range by the mapping rule from the MINPACK. - /// If the projection is not needed, set IsBounded = false befre calling the Minimization method. + /// Set parameters and bounds. /// /// The lower bounds of parameters. /// The upper bounds of parameters. - /// /// The scaling constants of parameters + /// The scaling constants of parameters /// The list to the parameters fix or free. - public void SetParameters(Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) + public void SetParameters(Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) { + if (initialGuess == null) + { + throw new ArgumentNullException("initialGuess"); + } + coefficients = initialGuess; + if (lowerBound != null && lowerBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) { throw new ArgumentException("The lower bounds must be finite."); - } + } + if (lowerBound != null && lowerBound.Count != initialGuess.Count) + { + throw new ArgumentException("The upper bounds can't have different elements from the initial guess."); + } LowerBound = lowerBound; if (upperBound != null && upperBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) { throw new ArgumentException("The upper bounds must be finite."); } - if (upperBound != null && lowerBound != null && upperBound.Count != lowerBound.Count) + if (upperBound != null && upperBound.Count != initialGuess.Count) { - throw new ArgumentException("The upper bounds can't have different elements from the lower bounds."); + throw new ArgumentException("The upper bounds can't have different elements from the initial guess."); } UpperBound = upperBound; @@ -295,13 +307,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { throw new ArgumentException("The scales must be finite."); } - if (scales != null && lowerBound != null && scales.Count != lowerBound.Count) - { - throw new ArgumentException("The upper bounds can't have different elements from the lower bounds."); - } - if (scales != null && upperBound != null && scales.Count != upperBound.Count) + if (scales != null && scales.Count != initialGuess.Count) { - throw new ArgumentException("The upper bounds can't have different elements from the upper bounds."); + throw new ArgumentException("The upper bounds can't have different elements from the initial guess."); } if (scales != null && scales.Count(x => x < 0) > 0) { @@ -309,48 +317,20 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels } Scales = scales; - IsBounded = (LowerBound != null || UpperBound != null || Scales != null); - - if (isFixed != null && lowerBound != null && isFixed.Count != lowerBound.Count) - { - throw new ArgumentException("The initial guess can't have different elements from the lower bounds."); - } - if (isFixed != null && upperBound != null && isFixed.Count != upperBound.Count) + if (isFixed != null && isFixed.Count != initialGuess.Count) { - throw new ArgumentException("The initial guess can't have different elements from the upper bounds."); - } - if (isFixed != null && scales != null && isFixed.Count != scales.Count) - { - throw new ArgumentException("The initial guess can't have different elements from the scales."); + throw new ArgumentException("The isFixed can't have different elements from the initial guess."); } if (isFixed != null && isFixed.Count(p => p == true) == isFixed.Count) { throw new ArgumentException("All the parameters can't be fixed."); } IsFixed = isFixed; - } - - /// - /// Set parameters. - /// - /// If bounded, the paramneters will be projected to unconstrained range by the mapping rule. - /// If the projection is not needed, set IsBounded = false befre calling the Minimization method. - /// - /// The lower bounds of parameters. - /// The upper bounds of parameters. - /// The scaling constants of parameters - /// The list to the parameters fix or free. - public void SetParameters(double[] lowerBound = null, double[] upperBound = null, double[] scales = null, bool[] isFixed = null) - { - var lb = (lowerBound == null) ? null : Vector.Build.DenseOfArray(lowerBound); - var ub = (upperBound == null) ? null : Vector.Build.DenseOfArray(upperBound); - var sc = (scales == null) ? null : Vector.Build.DenseOfArray(scales); - var fp = (isFixed == null) ? null : isFixed.ToList(); - SetParameters(lb, ub, sc, fp); + isBounded = LowerBound != null || UpperBound != null || Scales != null; } - public void EvaluateFunction(Vector parameters) + public void EvaluateAt(Vector parameters) { ValidateParameters(parameters); @@ -386,69 +366,84 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels // // Except when it is initial guess, the parameters argument is always internal parameter. // So, first map the parameters argument to the external parameters in order to calculate function values. - var Pext = (FunctionEvaluations > 0 && this.IsBounded) - ? ProjectParametersToExternal(parameters) - : parameters.Clone(); - // Project parameters, now this.Parameters are the internal parameters. - Parameters = (this.IsBounded) + Pext = (FunctionEvaluations > 0 && isBounded) + ? ProjectParametersToExternal(parameters) + : parameters.Clone(); + Pint = (isBounded) ? ProjectParametersToInternal(Pext) : Pext; + this.coefficients = Pint; + + if (IsFinished) + { + this.coefficients = Pext; + } + + hasFunctionValue = false; + hasJacobianValue = false; + + // don't keep references unnecessarily + jacobianValue = null; + gradientValue = null; + hessianValue = null; + } + + #region Private Methods + + private void EvaluateFunction() + { // Calculates the residuals, (y[i] - f(x[i]; p)) * L[i] - if (Values == null) + if (ModelValues == null) { - Values = Vector.Build.Dense(NumberOfObservations); + ModelValues = Vector.Build.Dense(NumberOfObservations); } for (int i = 0; i < NumberOfObservations; i++) { - Values[i] = userFunction(Pext, ObservedX[i]); + ModelValues[i] = userFunction(Pext, ObservedX[i]); } FunctionEvaluations++; // calculate the weighted residuals - Residuals = (Weights == null) - ? ObservedY - Values - : (ObservedY - Values).PointwiseMultiply(L); + residuals = (Weights == null) + ? ObservedY - ModelValues + : (ObservedY - ModelValues).PointwiseMultiply(L); // Calculate the residual sum of squares - Residue = Residuals.DotProduct(Residuals); + functionValue = residuals.DotProduct(residuals); return; } - public void EvaluateJacobian(Vector parameters) + private void EvaluateJacobian() { - var Pext = (IsBounded) - ? ProjectParametersToExternal(parameters) - : parameters.Clone(); - // Calculates the jacobian of x and p. if (userDerivatives != null) { // analytical jacobian - if (Jacobian == null) + if (jacobianValue == null) { - Jacobian = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); + jacobianValue = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); } for (int i = 0; i < NumberOfObservations; i++) { - Jacobian.SetRow(i, userDerivatives(Pext, ObservedX[i])); + jacobianValue.SetRow(i, userDerivatives(Pext, ObservedX[i])); } JacobianEvaluations++; } else { // numerical jacobian - Jacobian = NumericalJacobian(Pext, Values, AccuracyOrder); - FunctionEvaluations += AccuracyOrder; + jacobianValue = NumericalJacobian(Pext, ModelValues, accuracyOrder); + FunctionEvaluations += accuracyOrder; } - var scaleFactors = (this.IsBounded) - ? ScaleFactorsOfJacobian(Parameters) - : Vector.Build.Dense(Parameters.Count, 1.0); + var scaleFactors = (isBounded && !IsFinished) + ? ScaleFactorsOfJacobian(Pint) + : Vector.Build.Dense(Pint.Count, 1.0); - // Jint(x; Pint) = Jext(x; Pext) * scale where scale = dPext/dPint + // project jacobian: Jint(x; Pint) = Jext(x; Pext) * scale where scale = dPext/dPint for (int i = 0; i < NumberOfObservations; i++) { for (int j = 0; j < NumberOfParameters; j++) @@ -456,58 +451,24 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels if (IsFixed != null && IsFixed[j]) { // if j-th parameter is fixed, set J[i, j] = 0 - Jacobian[i, j] = 0.0; + jacobianValue[i, j] = 0.0; } else { - Jacobian[i, j] = Jacobian[i, j] * scaleFactors[j]; + jacobianValue[i, j] = jacobianValue[i, j] * scaleFactors[j]; } } } // Gradient, g = -J'W(y − f(x; p)) = -J'L(L'E) = -J'LR - Gradient = (Weights == null) - ? -Jacobian.Transpose() * (ObservedY - Values) - : -Jacobian.Transpose() * Weights * (ObservedY - Values); + gradientValue = (Weights == null) + ? -jacobianValue.Transpose() * (ObservedY - ModelValues) + : -jacobianValue.Transpose() * Weights * (ObservedY - ModelValues); // approximated Hessian, H = J'WJ + ∑LRiHi ~ J'WJ near the minimum - Hessian = (Weights == null) - ? Jacobian.Transpose() * Jacobian - : Jacobian.Transpose() * Weights * Jacobian; - } - - public void EvaluateCovariance(Vector parameters) - { - // convert to bounded(external) parameters - var Pext = (IsBounded) - ? ProjectParametersToExternal(parameters) - : parameters.Clone(); - - // set IsBounded = false to get external Parameters and covariance matrix - this.IsBounded = false; - - EvaluateFunction(Pext); - EvaluateJacobian(Pext); - - // restore isBounded - this.IsBounded = (LowerBound != null || UpperBound != null); - - if (Hessian == null || Residuals == null || DegreeOfFreedom < 1) - { - Covariance = null; - Correlation = null; - return; - } - - var covariance = Hessian.PseudoInverse() * Residuals.DotProduct(Residuals) / DegreeOfFreedom; - Covariance = covariance; - - var correlation = covariance.Clone(); - var d = correlation.Diagonal().PointwiseSqrt(); - var dd = d.OuterProduct(d); - Correlation = correlation.PointwiseDivide(dd); - - return; + hessianValue = (Weights == null) + ? jacobianValue.Transpose() * jacobianValue + : jacobianValue.Transpose() * Weights * jacobianValue; } private void ValidateParameters(Vector parameters) @@ -538,16 +499,13 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels } } - #region Numerical Derivatives - - // Numerical derivatives by using the central or forward finite difference - private Matrix NumericalJacobian(Vector parameters, Vector currentValues, int accuracyOrder = 2) + private Matrix NumericalJacobian(Vector Pext, Vector currentValues, int accuracyOrder = 2) { const double sqrtEpsilon = 1.4901161193847656250E-8; // sqrt(machineEpsilon) Matrix derivertives = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); - var d = 0.000003 * parameters.PointwiseAbs().PointwiseMaximum(sqrtEpsilon); + var d = 0.000003 * Pext.PointwiseAbs().PointwiseMaximum(sqrtEpsilon); var h = Vector.Build.Dense(NumberOfParameters); for (int i = 0; i < NumberOfObservations; i++) @@ -560,12 +518,12 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels if (accuracyOrder >= 6) { // f'(x) = {- f(x - 3h) + 9f(x - 2h) - 45f(x - h) + 45f(x + h) - 9f(x + 2h) + f(x + 3h)} / 60h + O(h^6) - var f1 = userFunction(parameters - 3 * h, x); - var f2 = userFunction(parameters - 2 * h, x); - var f3 = userFunction(parameters - h, x); - var f4 = userFunction(parameters + h, x); - var f5 = userFunction(parameters + 2 * h, x); - var f6 = userFunction(parameters + 3 * h, x); + var f1 = userFunction(Pext - 3 * h, x); + var f2 = userFunction(Pext - 2 * h, x); + var f3 = userFunction(Pext - h, x); + var f4 = userFunction(Pext + h, x); + var f5 = userFunction(Pext + 2 * h, x); + var f6 = userFunction(Pext + 3 * h, x); var prime = (-f1 + 9 * f2 - 45 * f3 + 45 * f4 - 9 * f5 + f6) / (60 * h[j]); derivertives[i, j] = prime; @@ -574,11 +532,11 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { // f'(x) = {-137f(x) + 300f(x + h) - 300f(x + 2h) + 200f(x + 3h) - 75f(x + 4h) + 12f(x + 5h)} / 60h + O(h^5) var f1 = currentValues[i]; - var f2 = userFunction(parameters + h, x); - var f3 = userFunction(parameters + 2 * h, x); - var f4 = userFunction(parameters + 3 * h, x); - var f5 = userFunction(parameters + 4 * h, x); - var f6 = userFunction(parameters + 5 * h, x); + var f2 = userFunction(Pext + h, x); + var f3 = userFunction(Pext + 2 * h, x); + var f4 = userFunction(Pext + 3 * h, x); + var f5 = userFunction(Pext + 4 * h, x); + var f6 = userFunction(Pext + 5 * h, x); var prime = (-137 * f1 + 300 * f2 - 300 * f3 + 200 * f4 - 75 * f5 + 12 * f6) / (60 * h[j]); derivertives[i, j] = prime; @@ -586,10 +544,10 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels else if (accuracyOrder == 4) { // f'(x) = {f(x - 2h) - 8f(x - h) + 8f(x + h) - f(x + 2h)} / 12h + O(h^4) - var f1 = userFunction(parameters - 2 * h, x); - var f2 = userFunction(parameters - h, x); - var f3 = userFunction(parameters + h, x); - var f4 = userFunction(parameters + 2 * h, x); + var f1 = userFunction(Pext - 2 * h, x); + var f2 = userFunction(Pext - h, x); + var f3 = userFunction(Pext + h, x); + var f4 = userFunction(Pext + 2 * h, x); var prime = (f1 - 8 * f2 + 8 * f3 - f4) / (12 * h[j]); derivertives[i, j] = prime; @@ -598,9 +556,9 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { // f'(x) = {-11f(x) + 18f(x + h) - 9f(x + 2h) + 2f(x + 3h)} / 6h + O(h^3) var f1 = currentValues[i]; - var f2 = userFunction(parameters + h, x); - var f3 = userFunction(parameters + 2 * h, x); - var f4 = userFunction(parameters + 3 * h, x); + var f2 = userFunction(Pext + h, x); + var f3 = userFunction(Pext + 2 * h, x); + var f4 = userFunction(Pext + 3 * h, x); var prime = (-11 * f1 + 18 * f2 - 9 * f3 + 2 * f4) / (6 * h[j]); derivertives[i, j] = prime; @@ -608,8 +566,8 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels else if (accuracyOrder == 2) { // f'(x) = {f(x + h) - f(x - h)} / 2h + O(h^2) - var f1 = userFunction(parameters + h, x); - var f2 = userFunction(parameters - h, x); + var f1 = userFunction(Pext + h, x); + var f2 = userFunction(Pext - h, x); var prime = (f1 - f2) / (2 * h[j]); derivertives[i, j] = prime; @@ -618,7 +576,7 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels { // f'(x) = {- f(x) + f(x + h)} / h + O(h) var f1 = currentValues[i]; - var f2 = userFunction(parameters + h, x); + var f2 = userFunction(Pext + h, x); var prime = (-f1 + f2) / h[j]; derivertives[i, j] = prime; @@ -631,10 +589,6 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels return derivertives; } - #endregion Numerical Derivatives - - #region Projection - private Vector ProjectParametersToInternal(Vector Pext) { var Pint = Pext.Clone(); @@ -771,6 +725,6 @@ namespace MathNet.Numerics.Optimization.ObjectiveModels return scale; } - #endregion Projection + #endregion Private Methods } } diff --git a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs index eb62acef..ce1ac838 100644 --- a/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/DogLegSubproblem.cs @@ -13,12 +13,10 @@ namespace MathNet.Numerics.Optimization.Subproblems var Gradient = objective.Gradient; var Hessian = objective.Hessian; - // newton point - // the Gauss–Newton step by solving the normal equations + // newton point, the Gauss–Newton step by solving the normal equations var Pgn = -Hessian.PseudoInverse() * Gradient; // Hessian.Solve(Gradient) fails so many times... - // cauchy point - // steepest descent direction is given by + // cauchy point, steepest descent direction is given by var alpha = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); var Psd = -alpha * Gradient; diff --git a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs index ec26b34a..73149818 100644 --- a/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs +++ b/src/Numerics/Optimization/Subproblems/NewtonCGSubproblem.cs @@ -58,7 +58,7 @@ namespace MathNet.Numerics.Optimization.Subproblems z = znext; r = rnext; - d = -rnext + rnext_sq / r_sq * d; ; + d = -rnext + rnext_sq / r_sq * d; } } } diff --git a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs index 22de97d8..d32c68f8 100644 --- a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs +++ b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs @@ -1,5 +1,7 @@ using MathNet.Numerics.LinearAlgebra; using System; +using System.Collections.Generic; +using System.Linq; namespace MathNet.Numerics.Optimization { @@ -46,39 +48,50 @@ namespace MathNet.Numerics.Optimization MaximumIterations = maximumIterations; } - public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess, + Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) { if (objective == null) throw new ArgumentNullException("objective"); if (initialGuess == null) throw new ArgumentNullException("initialGuess"); - return Minimum(objective, initialGuess, Subproblem, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); + return Minimum(Subproblem, objective, initialGuess, lowerBound, upperBound, scales, isFixed, + GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); } - public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess) + public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess, + double[] lowerBound = null, double[] upperBound = null, double[] scales = null, bool[] isFixed = null) { if (objective == null) throw new ArgumentNullException("objective"); if (initialGuess == null) throw new ArgumentNullException("initialGuess"); - return Minimum(objective, CreateVector.DenseOfArray(initialGuess), Subproblem, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); + var lb = (lowerBound == null) ? null : CreateVector.Dense(lowerBound); + var ub = (upperBound == null) ? null : CreateVector.Dense(upperBound); + var sc = (scales == null) ? null : CreateVector.Dense(scales); + var fx = (isFixed == null) ? null : isFixed.ToList(); + + return Minimum(Subproblem, objective, CreateVector.DenseOfArray(initialGuess), lb, ub, sc, fx, + GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); } /// /// Non-linear least square fitting by the trust-region algorithm. /// /// The objective model, including function, jacobian, observations, and parameter bounds. + /// The subproblem /// The initial guess values. - /// The subproblem /// The stopping threshold for L2 norm of the residuals. /// The stopping threshold for infinity norm of the gradient vector. /// The stopping threshold for L2 norm of the change of parameters. /// The stopping threshold for trust region radius /// The max iterations. /// - public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, ITrustRegionSubproblem subproblem, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) + public static NonlinearMinimizationResult Minimum(ITrustRegionSubproblem subproblem, IObjectiveModel objective, Vector initialGuess, + Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null, + double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) { // Non-linear least square fitting by the trust-region algorithm. // @@ -123,16 +136,19 @@ namespace MathNet.Numerics.Optimization if (initialGuess == null) throw new ArgumentNullException("initialGuess"); + objective.SetParameters(initialGuess, lowerBound, upperBound, scales, isFixed); + ExitCondition exitCondition = ExitCondition.None; // Initialize objective objective.FunctionEvaluations = 0; objective.JacobianEvaluations = 0; + objective.IsFinished = false; // First, calculate function values and setup variables - objective.EvaluateFunction(initialGuess); - var P = objective.Parameters; // current parameters - var RSS = objective.Residue; // Residual Sum of Squares = R'R + objective.EvaluateAt(initialGuess); + var P = objective.Point; // current parameters + var RSS = objective.Value; // Residual Sum of Squares = R'R var RSSinit = RSS; // RSS at initial gussing parameters if (maximumIterations < 0) @@ -140,7 +156,7 @@ namespace MathNet.Numerics.Optimization maximumIterations = 200 * (initialGuess.Count + 1); } - // if R == NaN, stop + // if RSS == NaN, stop if (double.IsNaN(RSS)) { exitCondition = ExitCondition.InvalidValues; @@ -159,10 +175,9 @@ namespace MathNet.Numerics.Optimization exitCondition = ExitCondition.Converged; // SmallRSS } - // Evaluate projected Hessian, and gradient - objective.EvaluateJacobian(P); - var Hessian = objective.Hessian; + // evaluate projected gradient and Hessian var Gradient = objective.Gradient; + var Hessian = objective.Hessian; // if ||g||_oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance) @@ -172,8 +187,6 @@ namespace MathNet.Numerics.Optimization if (exitCondition != ExitCondition.None) { - // finalize - objective.EvaluateCovariance(P); return new NonlinearMinimizationResult(objective, -1, exitCondition); } @@ -191,7 +204,7 @@ namespace MathNet.Numerics.Optimization var Pstep = subproblem.Pstep; var hitBoundary = subproblem.HitBoundary; // predicted reduction = L(0) - L(Δp) = -Δp'g - 1/2 * Δp'HΔp - var predictedReduction = -objective.Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(objective.Hessian * Pstep); + var predictedReduction = -Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(Hessian * Pstep); if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) { @@ -201,13 +214,20 @@ namespace MathNet.Numerics.Optimization var Pnew = P + Pstep; // parameters to test - objective.EvaluateFunction(Pnew); - var RSSnew = objective.Residue; + objective.EvaluateAt(Pnew); + var RSSnew = objective.Value; + + // if RSS == NaN, stop + if (double.IsNaN(RSSnew)) + { + exitCondition = ExitCondition.InvalidValues; + break; + } // calculate the ratio of the actual to the predicted reduction. double rho = (predictedReduction != 0) ? (RSS - RSSnew) / predictedReduction - : 0; + : 0.0; if (rho > 0.75 && hitBoundary) { @@ -229,8 +249,7 @@ namespace MathNet.Numerics.Optimization Pnew.CopyTo(P); RSS = RSSnew; - // update Jacobian, Hessian, and gradient - objective.EvaluateJacobian(P); + // evaluate projected gradient and Hessian Gradient = objective.Gradient; Hessian = objective.Hessian; @@ -253,9 +272,6 @@ namespace MathNet.Numerics.Optimization exitCondition = ExitCondition.ExceedIterations; } - // finalize - objective.EvaluateCovariance(P); - return new NonlinearMinimizationResult(objective, iterations, exitCondition); } } From 19bd5f576da4c75d0c1a088efc29d9ca9cee4356 Mon Sep 17 00:00:00 2001 From: diluculo Date: Fri, 4 Jan 2019 18:29:16 +0900 Subject: [PATCH 13/13] Reworked objective model and nonlinear minimizers to match the scheme of the existing minimizers. --- .../NonLinearCurveFittingTests.cs | 280 ++++--- src/Numerics/Optimization/IObjectiveModel.cs | 5 +- .../LevenbergMarquardtMinimizer.cs | 76 +- .../NonlinearMinimizationResult.cs | 28 +- .../Optimization/NonlinearMinimizerBase.cs | 302 ++++++++ .../Optimization/ObjectiveFunction.cs | 106 +++ .../NonlinearObjectiveFunction.cs | 436 +++++++++++ src/Numerics/Optimization/ObjectiveModel.cs | 56 -- .../ObjectiveModels/FittingObjectiveModel.cs | 730 ------------------ .../Optimization/TrustRegionMinimizerBase.cs | 81 +- 10 files changed, 1059 insertions(+), 1041 deletions(-) create mode 100644 src/Numerics/Optimization/NonlinearMinimizerBase.cs create mode 100644 src/Numerics/Optimization/ObjectiveFunctions/NonlinearObjectiveFunction.cs delete mode 100644 src/Numerics/Optimization/ObjectiveModel.cs delete mode 100644 src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs diff --git a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs index faae726c..2b7b553d 100644 --- a/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs +++ b/src/Numerics.Tests/OptimizationTests/NonLinearCurveFittingTests.cs @@ -19,16 +19,23 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // best fitted parameters: // a = 1 // b = 1 - private double RosenbrockModel(Vector p, double x) + private Vector RosenbrockModel(Vector p, Vector x) { - var y = Math.Pow(1.0 - p[0], 2) + 100.0 * Math.Pow(p[1] - p[0] * p[0], 2); + var y = CreateVector.Dense(x.Count); + for (int i = 0; i < x.Count; i++) + { + y[i] = Math.Pow(1.0 - p[0], 2) + 100.0 * Math.Pow(p[1] - p[0] * p[0], 2); + } return y; } - private Vector RosenbrockPrime(Vector p, double x) + private Matrix RosenbrockPrime(Vector p, Vector x) { - var prime = Vector.Build.Dense(p.Count); - prime[0] = 400.0 * p[0] * p[0] * p[0] - 400.0 * p[0] * p[1] + 2.0 * p[0] - 2.0; - prime[1] = 200.0 * (p[1] - p[0] * p[0]); + var prime = Matrix.Build.Dense(x.Count, p.Count); + for (int i = 0; i < x.Count; i++) + { + prime[i, 0] = 400.0 * p[0] * p[0] * p[0] - 400.0 * p[0] * p[1] + 2.0 * p[0] - 2.0; + prime[i, 1] = 200.0 * (p[1] - p[0] * p[0]); + } return prime; } private Vector RosenbrockX = Vector.Build.Dense(2); @@ -39,29 +46,27 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector RosebbrockLowerBound = new DenseVector(new double[] { -5.0, -5.0 }); private Vector RosenbrockUpperBound = new DenseVector(new double[] { 5.0, 5.0 }); - #endregion Rosenbrock - [Test] public void Rosenbrock_LM_Der() { // unconstrained - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + var obj = ObjectiveFunction.NonlinearModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); var result = solver.FindMinimum(obj, RosenbrockStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 2); } // box constrained - obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); + obj = ObjectiveFunction.NonlinearModel(RosenbrockModel, RosenbrockPrime, RosenbrockX, RosenbrockY); solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); result = solver.FindMinimum(obj, RosenbrockStart1, lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 2); } } @@ -69,52 +74,54 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests public void Rosenbrock_LM_Dif() { // unconstrained - var obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder:2); + var obj = ObjectiveFunction.NonlinearModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder:2); var solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); var result = solver.FindMinimum(obj, RosenbrockStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 2); } // box constrained - obj = ObjectiveModel.FittingModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); + obj = ObjectiveFunction.NonlinearModel(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); solver = new LevenbergMarquardtMinimizer(maximumIterations: 10000); result = solver.FindMinimum(obj, RosenbrockStart1, lowerBound: RosebbrockLowerBound, upperBound: RosenbrockUpperBound); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 2); } } [Test] public void Rosenbrock_Bfgs_Dif() { - var obj = ObjectiveModel.FittingFunction(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); - var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); + var obj = ObjectiveFunction.NonlinearFunction(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); + var solver = new BfgsMinimizer(1e-8, 1e-8, 1e-8, 1000); var result = solver.FindMinimum(obj, RosenbrockStart1); for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 2); } } [Test] public void Rosenbrock_LBfgs_Dif() { - var obj = ObjectiveModel.FittingFunction(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); - var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); + var obj = ObjectiveFunction.NonlinearFunction(RosenbrockModel, RosenbrockX, RosenbrockY, accuracyOrder: 6); + var solver = new LimitedMemoryBfgsMinimizer(1e-8, 1e-8, 1e-8, 1000); var result = solver.FindMinimum(obj, RosenbrockStart1); for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 3); + AssertHelpers.AlmostEqualRelative(RosenbrockPbest[i], result.MinimizingPoint[i], 2); } } + #endregion Rosenbrock + #region Rat43 // model: Rat43 (https://www.itl.nist.gov/div898/strd/nls/data/ratkowsky3.shtml) @@ -124,9 +131,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // b = 5.2771253025E+00 +/- 2.0828735829E+00 // c = 7.5962938329E-01 +/- 1.9566123451E-01 // d = 1.2792483859E+00 +/- 6.8761936385E-01 - private double Rat43Model(Vector p, double x) + private Vector Rat43Model(Vector p, Vector x) { - var y = p[0] / Math.Pow(1.0 + Math.Exp(p[1] - p[2] * x), 1.0 / p[3]); + var y = CreateVector.Dense(x.Count); + for (int i = 0; i < x.Count; i++) + { + y[i] = p[0] / Math.Pow(1.0 + Math.Exp(p[1] - p[2] * x[i]), 1.0 / p[3]); + } return y; } private Vector Rat43X = new DenseVector(new double[] { @@ -147,18 +158,16 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector Rat43Start1 = new DenseVector(new double[] { 100, 10, 1, 1 }); private Vector Rat43Start2 = new DenseVector(new double[] { 700, 5, 0.75, 1.3 }); - #endregion Rat43 - [Test] public void Rat43_LM_Dif() { - var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, Rat43Start1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(Rat43Pstd[i], result.StandardErrors[i], 6); } } @@ -166,13 +175,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Rat43_TRDL_Dif() { - var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); var result = solver.FindMinimum(obj, Rat43Start2); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.BestFitParameters[i], 2); + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.MinimizingPoint[i], 2); AssertHelpers.AlmostEqualRelative(Rat43Pstd[i], result.StandardErrors[i], 2); } } @@ -180,13 +189,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Rat43_TRNCG_Dif() { - var obj = ObjectiveModel.FittingModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); var result = solver.FindMinimum(obj, Rat43Start2); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.BestFitParameters[i], 2); + AssertHelpers.AlmostEqualRelative(Rat43Pbest[i], result.MinimizingPoint[i], 2); AssertHelpers.AlmostEqualRelative(Rat43Pstd[i], result.StandardErrors[i], 2); } } @@ -194,7 +203,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Rat43_Bfgs_Dif() { - var obj = ObjectiveModel.FittingFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, Rat43Start2); @@ -207,7 +216,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Rat43_LBfgs_Dif() { - var obj = ObjectiveModel.FittingFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearFunction(Rat43Model, Rat43X, Rat43Y, accuracyOrder: 6); var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, Rat43Start2); @@ -217,6 +226,8 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + #endregion Rat43 + #region BoxBod // model: BoxBod (https://www.itl.nist.gov/div898/strd/nls/data/boxbod.shtml) @@ -227,16 +238,23 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // best fitted parameters: // a = 2.1380940889E+02 +/- 1.2354515176E+01 // b = 5.4723748542E-01 +/- 1.0455993237E-01 - private double BoxBodModel(Vector p, double x) + private Vector BoxBodModel(Vector p, Vector x) { - var y = p[0] * (1.0 - Math.Exp(-p[1] * x)); + var y = CreateVector.Dense(x.Count); + for (int i = 0; i < x.Count; i++) + { + y[i] = p[0] * (1.0 - Math.Exp(-p[1] * x[i])); + } return y; } - private Vector BoxBodPrime(Vector p, double x) + private Matrix BoxBodPrime(Vector p, Vector x) { - var prime = Vector.Build.Dense(p.Count); - prime[0] = 1.0 - Math.Exp(-p[1] * x); - prime[1] = p[0] * x * Math.Exp(-p[1] * x); + var prime = Matrix.Build.Dense(x.Count, p.Count); + for (int i = 0; i < x.Count; i++) + { + prime[i, 0] = 1.0 - Math.Exp(-p[1] * x[i]); + prime[i, 1] = p[0] * x[i] * Math.Exp(-p[1] * x[i]); + } return prime; } private Vector BoxBodX = new DenseVector(new double[] { 1, 2, 3, 5, 7, 10 }); @@ -250,92 +268,90 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector BoxBodUpperBound = new DenseVector(new double[] { 1000.0, 100 }); private Vector BoxBodScales = new DenseVector(new double[] { 100.0, 0.1 }); - #endregion BoxBod - [Test] public void BoxBod_LM_Der() { // unconstrained - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + var obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, BoxBodStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // lower < parameters < upper // Note that in this case, scales have no effect. - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // lower < parameters, no scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // lower < parameters, scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound, scales: BoxBodScales); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // parameters < upper, no scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, upperBound: BoxBodUpperBound); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // parameters < upper, scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, upperBound: BoxBodUpperBound, scales: BoxBodScales); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // only scales - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, scales: BoxBodScales); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } } @@ -344,24 +360,24 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests public void BoxBod_LM_Dif() { // unconstrained - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder:6); + var obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder:6); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, BoxBodStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } // box constrained - obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); + obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); solver = new LevenbergMarquardtMinimizer(); result = solver.FindMinimum(obj, BoxBodStart1, lowerBound: BoxBodLowerBound, upperBound: BoxBodUpperBound); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 6); } } @@ -369,13 +385,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void BoxBod_TRDL_Dif() { - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); var result = solver.FindMinimum(obj, BoxBodStart1); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 3); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 3); } } @@ -383,14 +399,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void BoxBod_TRNCG_Dif() { - var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); - //var obj = ObjectiveModel.FittingModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(BoxBodModel, BoxBodX, BoxBodY, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); var result = solver.FindMinimum(obj, BoxBodStart2); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 3); AssertHelpers.AlmostEqualRelative(BoxBodPstd[i], result.StandardErrors[i], 3); } } @@ -398,7 +413,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void BoxBod_Bfgs_Der() { - var obj = ObjectiveModel.FittingFunction(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + var obj = ObjectiveFunction.NonlinearFunction(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 100); var result = solver.FindMinimum(obj, BoxBodStart2); @@ -408,6 +423,21 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests } } + [Test] + public void BoxBod_Newton_Der() + { + var obj = ObjectiveFunction.NonlinearFunction(BoxBodModel, BoxBodPrime, BoxBodX, BoxBodY); + var solver = new NewtonMinimizer(1e-10, 100); + var result = solver.FindMinimum(obj, BoxBodStart2); + + for (int i = 0; i < result.MinimizingPoint.Count; i++) + { + AssertHelpers.AlmostEqualRelative(BoxBodPbest[i], result.MinimizingPoint[i], 6); + } + } + + #endregion BoxBod + #region Thurber // model : Thurber (https://www.itl.nist.gov/div898/strd/nls/data/thurber.shtml) @@ -428,32 +458,38 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests // b5 = 9.6629502864E-01 +/- 3.1333340687E-02 // b6 = 3.9797285797E-01 +/- 1.4984928198E-02 // b7 = 4.9727297349E-02 +/- 6.5842344623E-03 - private double ThurberModel(Vector p, double x) + private Vector ThurberModel(Vector p, Vector x) { - var xSq = x * x; - var xCb = xSq * x; + var y = CreateVector.Dense(x.Count); + for (int i = 0; i < x.Count; i++) + { + var xSq = x[i] * x[i]; + var xCb = xSq * x[i]; - var y = (p[0] + p[1] * x + p[2] * xSq + p[3] * xCb) - / (1 + p[4] * x + p[5] * xSq + p[6] * xCb); + y[i] = (p[0] + p[1] * x[i] + p[2] * xSq + p[3] * xCb) + / (1 + p[4] * x[i] + p[5] * xSq + p[6] * xCb); + } return y; } - private Vector ThurberPrime(Vector p, double x) + private Matrix ThurberPrime(Vector p, Vector x) { - var prime = Vector.Build.Dense(p.Count); - - var xSq = x * x; - var xCb = xSq * x; - var num = (p[0] + x * (p[1] + x * (p[2] + p[3] * x))); - var den = (p[4] * x + p[5] * xSq + p[6] * xCb + 1.0); - var denSq = den * den; - - prime[0] = 1.0 / den; - prime[1] = x / den; - prime[2] = xSq / den; - prime[3] = xCb / den; - prime[4] = -(x * num) / denSq; - prime[5] = -(xSq * num) / denSq; - prime[6] = -(xCb * num) / denSq; + var prime = Matrix.Build.Dense(x.Count, p.Count); + for (int i = 0; i < x.Count; i++) + { + var xSq = x[i] * x[i]; + var xCb = xSq * x[i]; + var num = p[0] + x[i] * (p[1] + x[i] * (p[2] + p[3] * x[i])); + var den = p[4] * x[i] + p[5] * xSq + p[6] * xCb + 1.0; + var denSq = den * den; + + prime[i, 0] = 1.0 / den; + prime[i, 1] = x[i] / den; + prime[i, 2] = xSq / den; + prime[i, 3] = xCb / den; + prime[i, 4] = -(x[i] * num) / denSq; + prime[i, 5] = -(xSq * num) / denSq; + prime[i, 6] = -(xCb * num) / denSq; + } return prime; } private Vector ThurberX = new DenseVector(new double[] { @@ -485,18 +521,16 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests private Vector ThurberUpperBound = new DenseVector(new double[] { 1E6, 1E6, 1E6, 1E6, 1E6, 1E6, 1E6 }); private Vector ThurberScales = new DenseVector(new double[7] { 1000, 1000, 400, 40, 0.7, 0.3, 0.03 }); - #endregion Thurber - [Test] public void Thurber_LM_Der() { - var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberPrime, ThurberX, ThurberY); + var obj = ObjectiveFunction.NonlinearModel(ThurberModel, ThurberPrime, ThurberX, ThurberY); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, ThurberStart); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 6); } } @@ -504,13 +538,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_LM_Dif() { - var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LevenbergMarquardtMinimizer(); var result = solver.FindMinimum(obj, ThurberStart); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 6); + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 6); } } @@ -518,13 +552,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_TRDL_Dif() { - var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new TrustRegionDogLegMinimizer(); var result = solver.FindMinimum(obj, ThurberStart, scales: ThurberScales); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 3); AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 3); } } @@ -532,13 +566,13 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_TRNCG_Dif() { - var obj = ObjectiveModel.FittingModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearModel(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new TrustRegionNewtonCGMinimizer(); var result = solver.FindMinimum(obj, ThurberStart, scales: ThurberScales); - for (int i = 0; i < result.BestFitParameters.Count; i++) + for (int i = 0; i < result.MinimizingPoint.Count; i++) { - AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.BestFitParameters[i], 3); + AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 3); AssertHelpers.AlmostEqualRelative(ThurberPstd[i], result.StandardErrors[i], 3); } } @@ -546,7 +580,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_Bfgs_Dif() { - var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new BfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, ThurberStart); @@ -559,7 +593,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_BfgsB_Dif() { - var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new BfgsBMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, ThurberLowerBound, ThurberUpperBound, ThurberStart); @@ -572,7 +606,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests [Test] public void Thurber_LBfgs_Dif() { - var obj = ObjectiveModel.FittingFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); + var obj = ObjectiveFunction.NonlinearFunction(ThurberModel, ThurberX, ThurberY, accuracyOrder: 6); var solver = new LimitedMemoryBfgsMinimizer(1e-10, 1e-10, 1e-10, 1000); var result = solver.FindMinimum(obj, ThurberStart); @@ -581,5 +615,7 @@ namespace MathNet.Numerics.UnitTests.OptimizationTests AssertHelpers.AlmostEqualRelative(ThurberPbest[i], result.MinimizingPoint[i], 6); } } + + #endregion Thurber } } diff --git a/src/Numerics/Optimization/IObjectiveModel.cs b/src/Numerics/Optimization/IObjectiveModel.cs index b106a347..c9a7c66f 100644 --- a/src/Numerics/Optimization/IObjectiveModel.cs +++ b/src/Numerics/Optimization/IObjectiveModel.cs @@ -59,17 +59,14 @@ namespace MathNet.Numerics.Optimization bool IsGradientSupported { get; } bool IsHessianSupported { get; } - - bool IsFinished { get; set; } } public interface IObjectiveModel : IObjectiveModelEvaluation { - void SetParameters(Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null); + void SetParameters(Vector initialGuess, List isFixed = null); void EvaluateAt(Vector parameters); - /// Create a new independent copy of this objective function, evaluated at the same point. IObjectiveModel Fork(); IObjectiveFunction ToObjectiveFunction(); diff --git a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs index c39b0119..f340515d 100644 --- a/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs +++ b/src/Numerics/Optimization/LevenbergMarquardtMinimizer.cs @@ -5,55 +5,23 @@ using System.Linq; namespace MathNet.Numerics.Optimization { - public sealed class LevenbergMarquardtMinimizer + public class LevenbergMarquardtMinimizer : NonlinearMinimizerBase { - #region Tolerances and options - /// /// The scale factor for initial mu /// public static double InitialMu { get; set; } - /// - /// The stopping threshold for infinity norm of the gradient. - /// - public static double GradientTolerance { get; set; } - - /// - /// The stopping threshold for L2 norm of the change of the parameters. - /// - public static double StepTolerance { get; set; } - - /// - /// The stopping threshold for the function value or L2 norm of the residuals. - /// - public static double FunctionTolerance { get; set; } - - /// - /// The maximum number of iterations. - /// - public int MaximumIterations { get; set; } - - #endregion Tolerances and options - - public LevenbergMarquardtMinimizer(double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + public LevenbergMarquardtMinimizer(double initialMu = 1E-3, double gradientTolerance = 1E-15, double stepTolerance = 1E-15, double functionTolerance = 1E-15, int maximumIterations = -1) + : base(gradientTolerance, stepTolerance, functionTolerance, maximumIterations) { InitialMu = initialMu; - GradientTolerance = gradientTolerance; - StepTolerance = stepTolerance; - FunctionTolerance = functionTolerance; - MaximumIterations = maximumIterations; } public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) { - if (objective == null) - throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - - return Minimum(objective, initialGuess, lowerBound, upperBound, scales, isFixed, InitialMu, FunctionTolerance, GradientTolerance, StepTolerance, MaximumIterations); + return Minimum(objective, initialGuess, lowerBound, upperBound, scales, isFixed, InitialMu, GradientTolerance, StepTolerance, FunctionTolerance, MaximumIterations); } public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess, @@ -85,7 +53,7 @@ namespace MathNet.Numerics.Optimization /// The result of the Levenberg-Marquardt minimization public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null, - double initialMu = 1E-3, double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + double initialMu = 1E-3, double gradientTolerance = 1E-15, double stepTolerance = 1E-15, double functionTolerance = 1E-15, int maximumIterations = -1) { // Non-linear least square fitting by the Levenberg-Marduardt algorithm. // @@ -125,23 +93,16 @@ namespace MathNet.Numerics.Optimization if (objective == null) throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - - objective.SetParameters(initialGuess, lowerBound, upperBound, scales, isFixed); + ValidateBounds(initialGuess, lowerBound, upperBound, scales); + + objective.SetParameters(initialGuess, isFixed); ExitCondition exitCondition = ExitCondition.None; - // Initialize objective - objective.FunctionEvaluations = 0; - objective.JacobianEvaluations = 0; - objective.IsFinished = false; - // First, calculate function values and setup variables - objective.EvaluateAt(initialGuess); - var P = objective.Point; // current parameters + var P = ProjectToInternalParameters(initialGuess); // current internal parameters var Pstep = Vector.Build.Dense(P.Count); // the change of parameters - var RSS = objective.Value; // Residual Sum of Squares = R'R + var RSS = EvaluateFunction(objective, P); // Residual Sum of Squares = R'R if (maximumIterations < 0) { @@ -168,8 +129,9 @@ namespace MathNet.Numerics.Optimization } // Evaluate gradient and Hessian - var Gradient = objective.Gradient; - var Hessian = objective.Hessian; + var jac = EvaluateJacobian(objective, P); + var Gradient = jac.Item1; // objective.Gradient; + var Hessian = jac.Item2; // objective.Hessian; var diagonalOfHessian = Hessian.Diagonal(); // diag(H) // if ||g||oo <= gtol, found and stop @@ -200,14 +162,13 @@ namespace MathNet.Numerics.Optimization // if ||ΔP|| <= xTol * (||P|| + xTol), found and stop if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.DotProduct(P))) { - exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + exitCondition = ExitCondition.RelativePoints; break; } var Pnew = P + Pstep; // new parameters to test - - objective.EvaluateAt(Pnew); - var RSSnew = objective.Value; + // evaluate function at Pnew + var RSSnew = EvaluateFunction(objective, Pnew); if (double.IsNaN(RSSnew)) { @@ -229,8 +190,9 @@ namespace MathNet.Numerics.Optimization RSS = RSSnew; // update gradient and Hessian - Gradient = objective.Gradient; - Hessian = objective.Hessian; + jac = EvaluateJacobian(objective, P); + Gradient = jac.Item1; // objective.Gradient; + Hessian = jac.Item2; // objective.Hessian; diagonalOfHessian = Hessian.Diagonal(); // if ||g||_oo <= gtol, found and stop diff --git a/src/Numerics/Optimization/NonlinearMinimizationResult.cs b/src/Numerics/Optimization/NonlinearMinimizationResult.cs index 12381bbf..a67d3c85 100644 --- a/src/Numerics/Optimization/NonlinearMinimizationResult.cs +++ b/src/Numerics/Optimization/NonlinearMinimizationResult.cs @@ -1,8 +1,4 @@ using MathNet.Numerics.LinearAlgebra; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; namespace MathNet.Numerics.Optimization { @@ -13,7 +9,7 @@ namespace MathNet.Numerics.Optimization /// /// Returns the best fit parameters. /// - public Vector BestFitParameters { get { return ModelInfoAtMinimum.Point; } } + public Vector MinimizingPoint { get { return ModelInfoAtMinimum.Point; } } /// /// Returns the standard errors of the corresponding parameters @@ -23,17 +19,20 @@ namespace MathNet.Numerics.Optimization /// /// Returns the y-values of the fitted model that correspond to the independent values. /// - public Vector BestFitValues { get { return ModelInfoAtMinimum.ModelValues; } } + public Vector MinimizedValues { get { return ModelInfoAtMinimum.ModelValues; } } /// - /// Returns the residual sum of squares. + /// Returns the covariance matrix at minimizing point. /// - public double Residue { get { return ModelInfoAtMinimum.Value; } } - public double DegreeOfFreedom { get { return ModelInfoAtMinimum.DegreeOfFreedom; } } public Matrix Covariance { get; private set; } + + /// + /// Returns the correlation matrix at minimizing point. + /// public Matrix Correlation { get; private set; } public int Iterations { get; private set; } + public ExitCondition ReasonForExit { get; private set; } public NonlinearMinimizationResult(IObjectiveModel modelInfo, int iterations, ExitCondition reasonForExit) @@ -42,16 +41,15 @@ namespace MathNet.Numerics.Optimization Iterations = iterations; ReasonForExit = reasonForExit; - AnalyzeResult(modelInfo); + EvaluateCovariance(modelInfo); } - private void AnalyzeResult(IObjectiveModel objective) + private void EvaluateCovariance(IObjectiveModel objective) { - objective.IsFinished = true; - objective.EvaluateAt(objective.Point); + objective.EvaluateAt(objective.Point); // Hessian may be not yet updated. var Hessian = objective.Hessian; - if (Hessian == null || DegreeOfFreedom < 1) + if (Hessian == null || objective.DegreeOfFreedom < 1) { Covariance = null; Correlation = null; @@ -59,7 +57,7 @@ namespace MathNet.Numerics.Optimization return; } - Covariance = Hessian.PseudoInverse() * objective.Value / DegreeOfFreedom; + Covariance = Hessian.PseudoInverse() * objective.Value / objective.DegreeOfFreedom; if (Covariance != null) { diff --git a/src/Numerics/Optimization/NonlinearMinimizerBase.cs b/src/Numerics/Optimization/NonlinearMinimizerBase.cs new file mode 100644 index 00000000..1ff68382 --- /dev/null +++ b/src/Numerics/Optimization/NonlinearMinimizerBase.cs @@ -0,0 +1,302 @@ +using MathNet.Numerics.LinearAlgebra; +using System; +using System.Linq; + +namespace MathNet.Numerics.Optimization +{ + public abstract class NonlinearMinimizerBase + { + /// + /// The stopping threshold for the function value or L2 norm of the residuals. + /// + public static double FunctionTolerance { get; set; } + + /// + /// The stopping threshold for L2 norm of the change of the parameters. + /// + public static double StepTolerance { get; set; } + + /// + /// The stopping threshold for infinity norm of the gradient. + /// + public static double GradientTolerance { get; set; } + + /// + /// The maximum number of iterations. + /// + public static int MaximumIterations { get; set; } + + /// + /// The lower bound of the parameters. + /// + public static Vector LowerBound { get; private set; } + + /// + /// The upper bound of the parameters. + /// + public static Vector UpperBound { get; private set; } + + /// + /// The scale factors for the parameters. + /// + public static Vector Scales { get; private set; } + + private static bool IsBounded { get { return LowerBound != null || UpperBound != null || Scales != null; } } + + protected NonlinearMinimizerBase(double gradientTolerance = 1E-18, double stepTolerance = 1E-18, double functionTolerance = 1E-18, int maximumIterations = -1) + { + GradientTolerance = gradientTolerance; + StepTolerance = stepTolerance; + FunctionTolerance = functionTolerance; + MaximumIterations = maximumIterations; + } + + protected static void ValidateBounds(Vector parameters, Vector lowerBound = null, Vector upperBound = null, Vector scales = null) + { + if (parameters == null) + { + throw new ArgumentNullException("parameters"); + } + + if (lowerBound != null && lowerBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) + { + throw new ArgumentException("The lower bounds must be finite."); + } + if (lowerBound != null && lowerBound.Count != parameters.Count) + { + throw new ArgumentException("The lower bounds can't have different size from the parameters."); + } + LowerBound = lowerBound; + + if (upperBound != null && upperBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) + { + throw new ArgumentException("The upper bounds must be finite."); + } + if (upperBound != null && upperBound.Count != parameters.Count) + { + throw new ArgumentException("The upper bounds can't have different size from the parameetrs."); + } + UpperBound = upperBound; + + if (scales != null && scales.Count(x => double.IsInfinity(x) || double.IsNaN(x) || x == 0) > 0) + { + throw new ArgumentException("The scales must be finite."); + } + if (scales != null && scales.Count != parameters.Count) + { + throw new ArgumentException("The scales can't have different size from the parameters."); + } + if (scales != null && scales.Count(x => x < 0) > 0) + { + scales.PointwiseAbs(); + } + Scales = scales; + } + + protected static double EvaluateFunction(IObjectiveModel objective, Vector Pint) + { + var Pext = ProjectToExternalParameters(Pint); + objective.EvaluateAt(Pext); + return objective.Value; + } + + protected static Tuple, Matrix> EvaluateJacobian(IObjectiveModel objective, Vector Pint) + { + var gradient = objective.Gradient; + var hessian = objective.Hessian; + + if (IsBounded) + { + var scaleFactors = ScaleFactorsOfJacobian(Pint); // the parameters argument is always internal. + + for (int i = 0; i < gradient.Count; i++) + { + gradient[i] = gradient[i] * scaleFactors[i]; + } + + for (int i = 0; i < hessian.RowCount; i++) + { + for (int j = 0; j < hessian.ColumnCount; j++) + { + hessian[i, j] = hessian[i, j] * scaleFactors[i] * scaleFactors[j]; + } + } + } + + return new Tuple, Matrix>(gradient, hessian); + } + + #region Projection of Parameters + + // To handle the box constrained minimization as the unconstrained minimization, + // the parameters are mapping by the following rules, + // which are modified the rules shown in the ref[1] in order to introduce scales. + // + // 1. lower < Pext < upper + // Pint = asin(2 * (Pext - lower) / (upper - lower) - 1) + // Pext = lower + (sin(Pint) + 1) * (upper - lower) / 2 + // dPext/dPint = (upper - lower) / 2 * cos(Pint) + // + // 2. lower < Pext + // Pint = sqrt((Pext/scale - lower/scale + 1)^2 - 1) + // Pext = lower + scale * (sqrt(Pint^2 + 1) - 1) + // dPext/dPint = scale * Pint / sqrt(Pint^2 + 1) + // + // 3. Pext < upper + // Pint = sqrt((upper / scale - Pext / scale + 1)^2 - 1) + // Pext = upper + scale - scale * sqrt(Pint^2 + 1) + // dPext/dPint = - scale * Pint / sqrt(Pint^2 + 1) + // + // 4. no bounds, but scales + // Pint = Pext / scale + // Pext = Pint * scale + // dPext/dPint = scale + // + // The rules are applied in ProjectParametersToInternal, ProjectParametersToExternal, and ScaleFactorsOfJacobian methods. + // + // References: + // [1] https://lmfit.github.io/lmfit-py/bounds.html + // [2] MINUIT User's Guide, https://root.cern.ch/download/minuit.pdf + // + // Except when it is initial guess, the parameters argument is always internal parameter. + // So, first map the parameters argument to the external parameters in order to calculate function values. + + protected static Vector ProjectToInternalParameters(Vector Pext) + { + var Pint = Pext.Clone(); + + if (LowerBound != null && UpperBound != null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = Math.Asin((2.0 * (Pext[i] - LowerBound[i]) / (UpperBound[i] - LowerBound[i])) - 1.0); + } + + return Pint; + } + else if (LowerBound != null && UpperBound == null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = (Scales == null) + ? Math.Sqrt(Math.Pow(Pext[i] - LowerBound[i] + 1.0, 2) - 1.0) + : Math.Sqrt(Math.Pow((Pext[i] - LowerBound[i]) / Scales[i] + 1.0, 2) - 1.0); + } + + return Pint; + } + else if (LowerBound == null && UpperBound != null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = (Scales == null) + ? Math.Sqrt(Math.Pow(UpperBound[i] - Pext[i] + 1.0, 2) - 1.0) + : Math.Sqrt(Math.Pow((UpperBound[i] - Pext[i]) / Scales[i] + 1.0, 2) - 1.0); + } + + return Pint; + } + else if (Scales != null) + { + for (int i = 0; i < Pext.Count; i++) + { + Pint[i] = Pext[i] / Scales[i]; + } + + return Pint; + } + + return Pint; + } + + protected static Vector ProjectToExternalParameters(Vector Pint) + { + var Pext = Pint.Clone(); + + if (LowerBound != null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = LowerBound[i] + (UpperBound[i] / 2.0 - LowerBound[i] / 2.0) * (Math.Sin(Pint[i]) + 1.0); + } + + return Pext; + } + else if (LowerBound != null && UpperBound == null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = (Scales == null) + ? LowerBound[i] + Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0 + : LowerBound[i] + Scales[i] * (Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0); + } + + return Pext; + } + else if (LowerBound == null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = (Scales == null) + ? UpperBound[i] - Math.Sqrt(Pint[i] * Pint[i] + 1.0) + 1.0 + : UpperBound[i] - Scales[i] * (Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0); + } + + return Pext; + } + else if (Scales != null) + { + for (int i = 0; i < Pint.Count; i++) + { + Pext[i] = Pint[i] * Scales[i]; + } + + return Pext; + } + + return Pext; + } + + protected static Vector ScaleFactorsOfJacobian(Vector Pint) + { + var scale = Vector.Build.Dense(Pint.Count, 1.0); + + if (LowerBound != null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + scale[i] = (UpperBound[i] - LowerBound[i]) / 2.0 * Math.Cos(Pint[i]); + } + return scale; + } + else if (LowerBound != null && UpperBound == null) + { + for (int i = 0; i < Pint.Count; i++) + { + scale[i] = (Scales == null) + ? Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0) + : Scales[i] * Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); + } + return scale; + } + else if (LowerBound == null && UpperBound != null) + { + for (int i = 0; i < Pint.Count; i++) + { + scale[i] = (Scales == null) + ? -Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0) + : -Scales[i] * Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); + } + return scale; + } + else if (Scales != null) + { + return Scales; + } + + return scale; + } + + #endregion Projection of Parameters + } +} diff --git a/src/Numerics/Optimization/ObjectiveFunction.cs b/src/Numerics/Optimization/ObjectiveFunction.cs index a0c981d9..eaf1bfed 100644 --- a/src/Numerics/Optimization/ObjectiveFunction.cs +++ b/src/Numerics/Optimization/ObjectiveFunction.cs @@ -114,5 +114,111 @@ namespace MathNet.Numerics.Optimization { return new ScalarObjectiveFunction(function, derivative, secondDerivative); } + + /// + /// objective model with a user supplied jacobian for non-linear least squares regression. + /// + public static IObjectiveModel NonlinearModel(Func, Vector, Vector> function, + Func, Vector, Matrix> derivatives, + Vector observedX, Vector observedY, Vector weight = null) + { + var objective = new NonlinearObjectiveFunction(function, derivatives); + objective.SetObserved(observedX, observedY, weight); + return objective; + } + + /// + /// Objective model for non-linear least squares regression. + /// + public static IObjectiveModel NonlinearModel(Func, Vector, Vector> function, + Vector observedX, Vector observedY, Vector weight = null, + int accuracyOrder = 2) + { + var objective = new NonlinearObjectiveFunction(function, accuracyOrder: accuracyOrder); + objective.SetObserved(observedX, observedY, weight); + return objective; + } + + /// + /// Objective model with a user supplied jacobian for non-linear least squares regression. + /// + public static IObjectiveModel NonlinearModel(Func, double, double> function, + Func, double, Vector> derivatives, + Vector observedX, Vector observedY, Vector weight = null) + { + Vector func(Vector point, Vector x) + { + var functionValues = CreateVector.Dense(x.Count); + for (int i = 0; i < x.Count; i++) + { + functionValues[i] = function(point, x[i]); + } + + return functionValues; + } + + Matrix prime(Vector point, Vector x) + { + var derivativeValues = CreateMatrix.Dense(x.Count, point.Count); + for (int i = 0; i < x.Count; i++) + { + derivativeValues.SetRow(i, derivatives(point, x[i])); + } + + return derivativeValues; + } + + var objective = new NonlinearObjectiveFunction(func, prime); + objective.SetObserved(observedX, observedY, weight); + return objective; + } + + /// + /// Objective model for non-linear least squares regression. + /// + public static IObjectiveModel NonlinearModel(Func, double, double> function, + Vector observedX, Vector observedY, Vector weight = null, + int accuracyOrder = 2) + { + Vector func(Vector point, Vector x) + { + var functionValues = CreateVector.Dense(x.Count); + for (int i = 0; i < x.Count; i++) + { + functionValues[i] = function(point, x[i]); + } + + return functionValues; + } + + var objective = new NonlinearObjectiveFunction(func, accuracyOrder: accuracyOrder); + objective.SetObserved(observedX, observedY, weight); + return objective; + } + + /// + /// Objective function with a user supplied jacobian for nonlinear least squares regression. + /// + public static IObjectiveFunction NonlinearFunction(Func, Vector, Vector> function, + Func, Vector, Matrix> derivatives, + Vector observedX, Vector observedY, Vector weight = null) + { + var objective = new NonlinearObjectiveFunction(function, derivatives); + objective.SetObserved(observedX, observedY, weight); + return objective.ToObjectiveFunction(); + } + + /// + /// Objective function for nonlinear least squares regression. + /// The numerical jacobian with accuracy order is used. + /// + public static IObjectiveFunction NonlinearFunction(Func, Vector, Vector> function, + Vector observedX, Vector observedY, Vector weight = null, + int accuracyOrder = 2) + { + var objective = new NonlinearObjectiveFunction(function, null, accuracyOrder: accuracyOrder); + objective.SetObserved(observedX, observedY, weight); + return objective.ToObjectiveFunction(); + } } } diff --git a/src/Numerics/Optimization/ObjectiveFunctions/NonlinearObjectiveFunction.cs b/src/Numerics/Optimization/ObjectiveFunctions/NonlinearObjectiveFunction.cs new file mode 100644 index 00000000..7c5e36b0 --- /dev/null +++ b/src/Numerics/Optimization/ObjectiveFunctions/NonlinearObjectiveFunction.cs @@ -0,0 +1,436 @@ +using MathNet.Numerics.LinearAlgebra; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace MathNet.Numerics.Optimization.ObjectiveFunctions +{ + internal class NonlinearObjectiveFunction : IObjectiveModel + { + #region Private Variables + + readonly Func, Vector, Vector> userFunction; // (p, x) => f(x; p) + readonly Func, Vector, Matrix> userDerivative; // (p, x) => df(x; p)/dp + readonly int accuracyOrder; // the desired accuracy order to evaluate the jacobian by numerical approximaiton. + + Vector coefficients; + + bool hasFunctionValue; + double functionValue; // the residual sum of squares, residuals * residuals. + Vector residuals; // the weighted error values + + bool hasJacobianValue; + Matrix jacobianValue; // the Jacobian matrix. + Vector gradientValue; // the Gradient vector. + Matrix hessianValue; // the Hessian matrix. + + #endregion Private Variables + + #region Public Variables + + /// + /// Set or get the values of the independent variable. + /// + public Vector ObservedX { get; private set; } + + /// + /// Set or get the values of the observations. + /// + public Vector ObservedY { get; private set; } + + /// + /// Set or get the values of the weights for the observations. + /// + public Matrix Weights { get; private set; } + private Vector L; // Weights = LL' + + /// + /// Get whether parameters are fixed or free. + /// + public List IsFixed { get; private set; } + + /// + /// Get the number of observations. + /// + public int NumberOfObservations { get { return (ObservedY == null) ? 0 : ObservedY.Count; } } + + /// + /// Get the number of unknown parameters. + /// + public int NumberOfParameters { get { return (Point == null) ? 0 : Point.Count; } } + + /// + /// Get the degree of freedom + /// + public int DegreeOfFreedom + { + get + { + var df = NumberOfObservations - NumberOfParameters; + if (IsFixed != null) + { + df = df + IsFixed.Count(p => p == true); + } + return df; + } + } + + /// + /// Get the number of calls to function. + /// + public int FunctionEvaluations { get; set; } + + /// + /// Get the number of calls to jacobian. + /// + public int JacobianEvaluations { get; set; } + + #endregion Public Variables + + public NonlinearObjectiveFunction(Func, Vector, Vector> function, + Func, Vector, Matrix> derivative = null, int accuracyOrder = 2) + { + this.userFunction = function; + this.userDerivative = derivative; + this.accuracyOrder = Math.Min(6, Math.Max(1, accuracyOrder)); + } + + public IObjectiveModel Fork() + { + return new NonlinearObjectiveFunction(userFunction, userDerivative, accuracyOrder) + { + ObservedX = ObservedX, + ObservedY = ObservedY, + Weights = Weights, + + coefficients = coefficients, + + hasFunctionValue = hasFunctionValue, + functionValue = functionValue, + + hasJacobianValue = hasJacobianValue, + jacobianValue = jacobianValue, + gradientValue = gradientValue, + hessianValue = hessianValue + }; + } + + public IObjectiveModel CreateNew() + { + return new NonlinearObjectiveFunction(userFunction, userDerivative, accuracyOrder); + } + + /// + /// Set or get the values of the parameters. + /// + public Vector Point { get { return coefficients; } } + + /// + /// Get the y-values of the fitted model that correspond to the independent values. + /// + public Vector ModelValues { get; private set; } + + /// + /// Get the residual sum of squares. + /// + public double Value + { + get + { + if (!hasFunctionValue) + { + EvaluateFunction(); + hasFunctionValue = true; + } + return functionValue; + } + } + + /// + /// Get the Gradient vector of x and p. + /// + public Vector Gradient + { + get + { + if (!hasJacobianValue) + { + EvaluateJacobian(); + hasJacobianValue = true; + } + return gradientValue; + } + } + + /// + /// Get the Hessian matrix of x and p, J'WJ + /// + public Matrix Hessian + { + get + { + if (!hasJacobianValue) + { + EvaluateJacobian(); + hasJacobianValue = true; + } + return hessianValue; + } + } + + public bool IsGradientSupported { get { return true; } } + public bool IsHessianSupported { get { return true; } } + + /// + /// Set observed data to fit. + /// + public void SetObserved(Vector observedX, Vector observedY, Vector weights = null) + { + if (observedX == null || observedY == null) + { + throw new ArgumentNullException("The data set can't be null."); + } + if (observedX.Count != observedY.Count) + { + throw new ArgumentException("The observed x data can't have different from observed y data."); + } + ObservedX = observedX; + ObservedY = observedY; + + if (weights != null && weights.Count != observedY.Count) + { + throw new ArgumentException("The weightings can't have different from observations."); + } + if (weights != null && weights.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) + { + throw new ArgumentException("The weightings are not well-defined."); + } + if (weights != null && weights.Count(x => x == 0) == weights.Count) + { + throw new ArgumentException("All the weightings can't be zero."); + } + if (weights != null && weights.Count(x => x < 0) > 0) + { + weights = weights.PointwiseAbs(); + } + + Weights = (weights == null) + ? null + : Matrix.Build.DenseOfDiagonalVector(weights); + + L = (weights == null) + ? null + : Weights.Diagonal().PointwiseSqrt(); + } + + /// + /// Set parameters and bounds. + /// + /// The initial values of parameters. + /// The list to the parameters fix or free. + public void SetParameters(Vector initialGuess, List isFixed = null) + { + if (initialGuess == null) + { + throw new ArgumentNullException("initialGuess"); + } + coefficients = initialGuess; + + if (isFixed != null && isFixed.Count != initialGuess.Count) + { + throw new ArgumentException("The isFixed can't have different size from the initial guess."); + } + if (isFixed != null && isFixed.Count(p => p == true) == isFixed.Count) + { + throw new ArgumentException("All the parameters can't be fixed."); + } + IsFixed = isFixed; + } + + public void EvaluateAt(Vector parameters) + { + if (parameters == null) + { + throw new ArgumentNullException("parameters"); + } + if (parameters.Count(p => double.IsNaN(p) || double.IsInfinity(p)) > 0) + { + throw new ArgumentException("The parameters must be finite."); + } + + coefficients = parameters; + hasFunctionValue = false; + hasJacobianValue = false; + + jacobianValue = null; + gradientValue = null; + hessianValue = null; + } + + public IObjectiveFunction ToObjectiveFunction() + { + Tuple, Matrix> function(Vector point) + { + EvaluateAt(point); + + return new Tuple, Matrix>(Value, Gradient, Hessian); + } + + var objective = new GradientHessianObjectiveFunction(function); + return objective; + } + + #region Private Methods + + private void EvaluateFunction() + { + // Calculates the residuals, (y[i] - f(x[i]; p)) * L[i] + if (ModelValues == null) + { + ModelValues = Vector.Build.Dense(NumberOfObservations); + } + ModelValues = userFunction(Point, ObservedX); + FunctionEvaluations++; + + // calculate the weighted residuals + residuals = (Weights == null) + ? ObservedY - ModelValues + : (ObservedY - ModelValues).PointwiseMultiply(L); + + // Calculate the residual sum of squares + functionValue = residuals.DotProduct(residuals); + + return; + } + + private void EvaluateJacobian() + { + // Calculates the jacobian of x and p. + if (userDerivative != null) + { + // analytical jacobian + jacobianValue = userDerivative(Point, ObservedX); + JacobianEvaluations++; + } + else + { + // numerical jacobian + jacobianValue = NumericalJacobian(Point, ModelValues, accuracyOrder); + FunctionEvaluations += accuracyOrder; + } + + // weighted jacobian + for (int i = 0; i < NumberOfObservations; i++) + { + for (int j = 0; j < NumberOfParameters; j++) + { + if (IsFixed != null && IsFixed[j]) + { + // if j-th parameter is fixed, set J[i, j] = 0 + jacobianValue[i, j] = 0.0; + } + else + { + jacobianValue[i, j] = (Weights == null) + ? jacobianValue[i, j] + : jacobianValue[i, j] * L[j]; + } + } + } + + // Gradient, g = -J'W(y − f(x; p)) = -J'L(L'E) = -J'LR + gradientValue = -jacobianValue.Transpose() * residuals; + + // approximated Hessian, H = J'WJ + ∑LRiHi ~ J'WJ near the minimum + hessianValue = jacobianValue.Transpose() * jacobianValue; + } + + private Matrix NumericalJacobian(Vector parameters, Vector currentValues, int accuracyOrder = 2) + { + const double sqrtEpsilon = 1.4901161193847656250E-8; // sqrt(machineEpsilon) + + Matrix derivertives = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); + + var d = 0.000003 * parameters.PointwiseAbs().PointwiseMaximum(sqrtEpsilon); + + var h = Vector.Build.Dense(NumberOfParameters); + for (int j = 0; j < NumberOfParameters; j++) + { + h[j] = d[j]; + + if (accuracyOrder >= 6) + { + // f'(x) = {- f(x - 3h) + 9f(x - 2h) - 45f(x - h) + 45f(x + h) - 9f(x + 2h) + f(x + 3h)} / 60h + O(h^6) + var f1 = userFunction(parameters - 3 * h, ObservedX); + var f2 = userFunction(parameters - 2 * h, ObservedX); + var f3 = userFunction(parameters - h, ObservedX); + var f4 = userFunction(parameters + h, ObservedX); + var f5 = userFunction(parameters + 2 * h, ObservedX); + var f6 = userFunction(parameters + 3 * h, ObservedX); + + var prime = (-f1 + 9 * f2 - 45 * f3 + 45 * f4 - 9 * f5 + f6) / (60 * h[j]); + derivertives.SetColumn(j, prime); + } + else if (accuracyOrder == 5) + { + // f'(x) = {-137f(x) + 300f(x + h) - 300f(x + 2h) + 200f(x + 3h) - 75f(x + 4h) + 12f(x + 5h)} / 60h + O(h^5) + var f1 = currentValues; + var f2 = userFunction(parameters + h, ObservedX); + var f3 = userFunction(parameters + 2 * h, ObservedX); + var f4 = userFunction(parameters + 3 * h, ObservedX); + var f5 = userFunction(parameters + 4 * h, ObservedX); + var f6 = userFunction(parameters + 5 * h, ObservedX); + + var prime = (-137 * f1 + 300 * f2 - 300 * f3 + 200 * f4 - 75 * f5 + 12 * f6) / (60 * h[j]); + derivertives.SetColumn(j, prime); + } + else if (accuracyOrder == 4) + { + // f'(x) = {f(x - 2h) - 8f(x - h) + 8f(x + h) - f(x + 2h)} / 12h + O(h^4) + var f1 = userFunction(parameters - 2 * h, ObservedX); + var f2 = userFunction(parameters - h, ObservedX); + var f3 = userFunction(parameters + h, ObservedX); + var f4 = userFunction(parameters + 2 * h, ObservedX); + + var prime = (f1 - 8 * f2 + 8 * f3 - f4) / (12 * h[j]); + derivertives.SetColumn(j, prime); + } + else if (accuracyOrder == 3) + { + // f'(x) = {-11f(x) + 18f(x + h) - 9f(x + 2h) + 2f(x + 3h)} / 6h + O(h^3) + var f1 = currentValues; + var f2 = userFunction(parameters + h, ObservedX); + var f3 = userFunction(parameters + 2 * h, ObservedX); + var f4 = userFunction(parameters + 3 * h, ObservedX); + + var prime = (-11 * f1 + 18 * f2 - 9 * f3 + 2 * f4) / (6 * h[j]); + derivertives.SetColumn(j, prime); + } + else if (accuracyOrder == 2) + { + // f'(x) = {f(x + h) - f(x - h)} / 2h + O(h^2) + var f1 = userFunction(parameters + h, ObservedX); + var f2 = userFunction(parameters - h, ObservedX); + + var prime = (f1 - f2) / (2 * h[j]); + derivertives.SetColumn(j, prime); + } + else + { + // f'(x) = {- f(x) + f(x + h)} / h + O(h) + var f1 = currentValues; + var f2 = userFunction(parameters + h, ObservedX); + + var prime = (-f1 + f2) / h[j]; + derivertives.SetColumn(j, prime); + } + + h[j] = 0; + } + + return derivertives; + } + + #endregion Private Methods + } +} diff --git a/src/Numerics/Optimization/ObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModel.cs deleted file mode 100644 index 77cae337..00000000 --- a/src/Numerics/Optimization/ObjectiveModel.cs +++ /dev/null @@ -1,56 +0,0 @@ -using MathNet.Numerics.LinearAlgebra; -using MathNet.Numerics.Optimization.ObjectiveModels; -using System; - -namespace MathNet.Numerics.Optimization -{ - public static class ObjectiveModel - { - /// - /// Fitting model with a user supplied jacobian for non-linear least squares regression. - /// - public static IObjectiveModel FittingModel(Func, double, double> function, Func, double, Vector> derivatives, - Vector observedX, Vector observedY, Vector weight = null) - { - var objective = new FittingObjectiveModel(function, derivatives); - objective.SetObserved(observedX, observedY, weight); - return objective; - } - - /// - /// Fitting model for non-linear least squares regression. - /// - public static IObjectiveModel FittingModel(Func, double, double> function, - Vector observedX, Vector observedY, Vector weight = null, - int accuracyOrder = 2) - { - var objective = new FittingObjectiveModel(function, accuracyOrder: accuracyOrder); - objective.SetObserved(observedX, observedY, weight); - return objective; - } - - /// - /// Fitting function with a user supplied jacobian for nonlinear least squares regression by the line search algorithm. - /// - public static IObjectiveFunction FittingFunction(Func, double, double> function, Func, double, Vector> derivatives, - Vector observedX, Vector observedY, Vector weight = null) - { - var objective = new FittingObjectiveModel(function, derivatives); - objective.SetObserved(observedX, observedY, weight); - return objective.ToObjectiveFunction(); - } - - /// - /// Fitting function for nonlinear least squares regression by the line search algorithm. - /// The numerical jacobian with accuracy order is used. - /// - public static IObjectiveFunction FittingFunction(Func, double, double> function, - Vector observedX, Vector observedY, Vector weight = null, - int accuracyOrder = 2) - { - var objective = new FittingObjectiveModel(function, null, accuracyOrder: accuracyOrder); - objective.SetObserved(observedX, observedY, weight); - return objective.ToObjectiveFunction(); - } - } -} diff --git a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs b/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs deleted file mode 100644 index eb30f357..00000000 --- a/src/Numerics/Optimization/ObjectiveModels/FittingObjectiveModel.cs +++ /dev/null @@ -1,730 +0,0 @@ -using MathNet.Numerics.LinearAlgebra; -using MathNet.Numerics.Optimization.ObjectiveFunctions; -using System; -using System.Collections.Generic; -using System.Linq; - -namespace MathNet.Numerics.Optimization.ObjectiveModels -{ - internal class FittingObjectiveModel : IObjectiveModel - { - #region Private Variables - - readonly Func, double, double> userFunction; // (p, x) => f(x; p) - readonly Func, double, Vector> userDerivatives; // (p, x) => df(x; p)/dp - readonly int accuracyOrder; // the desired accuracy order to evaluate the jacobian by numerical approximaiton. - - Vector coefficients; - Vector Pint; // internal(unbounded) coefficients - public Vector Pext; // external(bounded) coefficients - - bool hasFunctionValue; - double functionValue; // the residual sum of squares. Residuals * Residuals - Vector residuals; // the error values - - bool hasJacobianValue; - Matrix jacobianValue; // the Jacobian matrix. - Vector gradientValue; // the Gradient vector. - Matrix hessianValue; // the Hessian matrix. - - bool isBounded; - - #endregion Private Variables - - #region Public Variables - Observed Data - - /// - /// Set or get the values of the independent variable. - /// - public Vector ObservedX { get; private set; } - - /// - /// Set or get the values of the observations. - /// - public Vector ObservedY { get; private set; } - - /// - /// Set or get the values of the weights for the observations. - /// - public Matrix Weights { get; private set; } - private Vector L; // Weights = LL' - - /// - /// Get the number of observations. - /// - public int NumberOfObservations { get { return (ObservedY == null) ? 0 : ObservedY.Count; } } - - #endregion Public Variables - Observed Data - - #region Public Variables - Bounds of Parameter - - /// - /// Get the values of the parameters. - /// - public List IsFixed { get; private set; } - - /// - /// Get the values of the parameters. - /// - public Vector LowerBound { get; private set; } - - /// - /// Get the values of the parameters. - /// - public Vector UpperBound { get; private set; } - - /// - /// Get the scale factor of the parameters. - /// - public Vector Scales { get; private set; } - - /// - /// Get the number of unknown parameters. - /// - public int NumberOfParameters { get { return (Point == null) ? 0 : Point.Count; } } - - #endregion Public Variables - Bounds of Parameter - - #region Public Variables - Others - - /// - /// Get the number of calls to function. - /// - public int FunctionEvaluations { get; set; } - /// - /// Get the number of calls to jacobian. - /// - public int JacobianEvaluations { get; set; } - - #endregion Public Variables - Others - - public FittingObjectiveModel(Func, double, double>function, Func, double, Vector> derivatives = null, int accuracyOrder = 2) - { - this.userFunction = function; - this.userDerivatives = derivatives; - this.accuracyOrder = Math.Min(6, Math.Max(1, accuracyOrder)); - - IsFinished = false; - } - - public IObjectiveModel Fork() - { - return new FittingObjectiveModel(userFunction, userDerivatives, accuracyOrder) - { - ObservedX = ObservedX, - ObservedY = ObservedY, - Weights = Weights, - - coefficients = coefficients, - Pint = Pint, - Pext = Pext, - - hasFunctionValue = hasFunctionValue, - functionValue = functionValue, - - hasJacobianValue = hasJacobianValue, - jacobianValue = jacobianValue, - gradientValue = gradientValue, - hessianValue = hessianValue - }; - } - - public IObjectiveModel CreateNew() - { - return new FittingObjectiveModel(userFunction, userDerivatives, accuracyOrder); - } - - /// - /// Set or get the values of the parameters. - /// - public Vector Point { get { return coefficients; } } - - /// - /// Get the y-values of the fitted model that correspond to the independent values. - /// - public Vector ModelValues { get; private set; } - - /// - /// Get the residual sum of squares. - /// - public double Value - { - get - { - if (!hasFunctionValue) - { - EvaluateFunction(); - hasFunctionValue = true; - } - return functionValue; - } - } - - /// - /// Get the Gradient vector of x and p. - /// - public Vector Gradient - { - get - { - if (!hasJacobianValue) - { - EvaluateJacobian(); - hasJacobianValue = true; - } - return gradientValue; - } - } - - /// - /// Get the Hessian matrix of x and p, J'WJ - /// - public Matrix Hessian - { - get - { - if (!hasJacobianValue) - { - EvaluateJacobian(); - hasJacobianValue = true; - } - return hessianValue; - } - } - - /// - /// Get the degree of freedom - /// - public int DegreeOfFreedom - { - get - { - var df = NumberOfObservations - NumberOfParameters; - if (IsFixed != null) - { - df = df + IsFixed.Count(p => p == true); - } - return df; - } - } - - public bool IsGradientSupported { get { return true; } } - public bool IsHessianSupported { get { return true; } } - - public bool IsFinished { get; set; } - - public IObjectiveFunction ToObjectiveFunction() - { - Tuple, Matrix> function(Vector point) - { - EvaluateAt(point); - - return new Tuple, Matrix>(Value, Gradient, Hessian); - } - - var objective = new GradientHessianObjectiveFunction(function); - return objective; - } - - /// - /// Set observed data to fit. - /// - public void SetObserved(Vector observedX, Vector observedY, Vector weights = null) - { - if (observedX == null || observedY == null) - { - throw new ArgumentNullException("The data set can't be null."); - } - if (observedX.Count != observedY.Count) - { - throw new ArgumentException("The observed x data can't have different from observed y data."); - } - ObservedX = observedX; - ObservedY = observedY; - - if (weights != null && weights.Count != observedY.Count) - { - throw new ArgumentException("The weightings can't have different from observations."); - } - if (weights != null && weights.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) - { - throw new ArgumentException("The weightings are not well-defined."); - } - if (weights != null && weights.Count(x => x == 0) == weights.Count) - { - throw new ArgumentException("All the weightings can't be zero."); - } - if (weights != null && weights.Count(x => x < 0) > 0) - { - weights = weights.PointwiseAbs(); - } - - Weights = (weights == null) - ? null - : Matrix.Build.DenseOfDiagonalVector(weights); - - L = (weights == null) - ? null - : Weights.Diagonal().PointwiseSqrt(); - } - - /// - /// Set parameters and bounds. - /// - /// The lower bounds of parameters. - /// The upper bounds of parameters. - /// The scaling constants of parameters - /// The list to the parameters fix or free. - public void SetParameters(Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) - { - if (initialGuess == null) - { - throw new ArgumentNullException("initialGuess"); - } - coefficients = initialGuess; - - if (lowerBound != null && lowerBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) - { - throw new ArgumentException("The lower bounds must be finite."); - } - if (lowerBound != null && lowerBound.Count != initialGuess.Count) - { - throw new ArgumentException("The upper bounds can't have different elements from the initial guess."); - } - LowerBound = lowerBound; - - if (upperBound != null && upperBound.Count(x => double.IsInfinity(x) || double.IsNaN(x)) > 0) - { - throw new ArgumentException("The upper bounds must be finite."); - } - if (upperBound != null && upperBound.Count != initialGuess.Count) - { - throw new ArgumentException("The upper bounds can't have different elements from the initial guess."); - } - UpperBound = upperBound; - - if (scales != null && scales.Count(x => double.IsInfinity(x) || double.IsNaN(x) || x == 0) > 0) - { - throw new ArgumentException("The scales must be finite."); - } - if (scales != null && scales.Count != initialGuess.Count) - { - throw new ArgumentException("The upper bounds can't have different elements from the initial guess."); - } - if (scales != null && scales.Count(x => x < 0) > 0) - { - scales.PointwiseAbs(); - } - Scales = scales; - - if (isFixed != null && isFixed.Count != initialGuess.Count) - { - throw new ArgumentException("The isFixed can't have different elements from the initial guess."); - } - if (isFixed != null && isFixed.Count(p => p == true) == isFixed.Count) - { - throw new ArgumentException("All the parameters can't be fixed."); - } - IsFixed = isFixed; - - isBounded = LowerBound != null || UpperBound != null || Scales != null; - } - - public void EvaluateAt(Vector parameters) - { - ValidateParameters(parameters); - - // To handle the box constrained minimization as the unconstrained minimization, - // the parameters are mapping by the following rules, - // which are modified the rules shown in the ref[1] in order to introduce scales. - // - // 1. lower < Pext < upper - // Pint = asin(2 * (Pext - lower) / (upper - lower) - 1) - // Pext = lower + (sin(Pint) + 1) * (upper - lower) / 2 - // dPext/dPint = (upper - lower) / 2 * cos(Pint) - // - // 2. lower < Pext - // Pint = sqrt((Pext/scale - lower/scale + 1)^2 - 1) - // Pext = lower + scale * (sqrt(Pint^2 + 1) - 1) - // dPext/dPint = scale * Pint / sqrt(Pint^2 + 1) - // - // 3. Pext < upper - // Pint = sqrt((upper / scale - Pext / scale + 1)^2 - 1) - // Pext = upper + scale - scale * sqrt(Pint^2 + 1) - // dPext/dPint = - scale * Pint / sqrt(Pint^2 + 1) - // - // 4. no bounds, but scales - // Pint = Pext / scale - // Pext = Pint * scale - // dPext/dPint = scale - // - // The rules are applied in ProjectParametersToInternal, ProjectParametersToExternal, and ScaleFactorsOfJacobian methods. - // - // References: - // [1] https://lmfit.github.io/lmfit-py/bounds.html - // [2] MINUIT User's Guide, https://root.cern.ch/download/minuit.pdf - // - // Except when it is initial guess, the parameters argument is always internal parameter. - // So, first map the parameters argument to the external parameters in order to calculate function values. - - Pext = (FunctionEvaluations > 0 && isBounded) - ? ProjectParametersToExternal(parameters) - : parameters.Clone(); - Pint = (isBounded) - ? ProjectParametersToInternal(Pext) - : Pext; - - this.coefficients = Pint; - - if (IsFinished) - { - this.coefficients = Pext; - } - - hasFunctionValue = false; - hasJacobianValue = false; - - // don't keep references unnecessarily - jacobianValue = null; - gradientValue = null; - hessianValue = null; - } - - #region Private Methods - - private void EvaluateFunction() - { - // Calculates the residuals, (y[i] - f(x[i]; p)) * L[i] - if (ModelValues == null) - { - ModelValues = Vector.Build.Dense(NumberOfObservations); - } - for (int i = 0; i < NumberOfObservations; i++) - { - ModelValues[i] = userFunction(Pext, ObservedX[i]); - } - FunctionEvaluations++; - - // calculate the weighted residuals - residuals = (Weights == null) - ? ObservedY - ModelValues - : (ObservedY - ModelValues).PointwiseMultiply(L); - - // Calculate the residual sum of squares - functionValue = residuals.DotProduct(residuals); - - return; - } - - private void EvaluateJacobian() - { - // Calculates the jacobian of x and p. - if (userDerivatives != null) - { - // analytical jacobian - if (jacobianValue == null) - { - jacobianValue = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); - } - for (int i = 0; i < NumberOfObservations; i++) - { - jacobianValue.SetRow(i, userDerivatives(Pext, ObservedX[i])); - } - JacobianEvaluations++; - } - else - { - // numerical jacobian - jacobianValue = NumericalJacobian(Pext, ModelValues, accuracyOrder); - FunctionEvaluations += accuracyOrder; - } - - var scaleFactors = (isBounded && !IsFinished) - ? ScaleFactorsOfJacobian(Pint) - : Vector.Build.Dense(Pint.Count, 1.0); - - // project jacobian: Jint(x; Pint) = Jext(x; Pext) * scale where scale = dPext/dPint - for (int i = 0; i < NumberOfObservations; i++) - { - for (int j = 0; j < NumberOfParameters; j++) - { - if (IsFixed != null && IsFixed[j]) - { - // if j-th parameter is fixed, set J[i, j] = 0 - jacobianValue[i, j] = 0.0; - } - else - { - jacobianValue[i, j] = jacobianValue[i, j] * scaleFactors[j]; - } - } - } - - // Gradient, g = -J'W(y − f(x; p)) = -J'L(L'E) = -J'LR - gradientValue = (Weights == null) - ? -jacobianValue.Transpose() * (ObservedY - ModelValues) - : -jacobianValue.Transpose() * Weights * (ObservedY - ModelValues); - - // approximated Hessian, H = J'WJ + ∑LRiHi ~ J'WJ near the minimum - hessianValue = (Weights == null) - ? jacobianValue.Transpose() * jacobianValue - : jacobianValue.Transpose() * Weights * jacobianValue; - } - - private void ValidateParameters(Vector parameters) - { - if (parameters == null) - { - throw new ArgumentNullException("parameters"); - } - else if (parameters.Count(p => double.IsNaN(p) || double.IsInfinity(p)) > 0) - { - throw new ArgumentException("the parameters must be finite."); - } - if (LowerBound != null && parameters.Count != LowerBound.Count) - { - throw new ArgumentException("The parameters can't have different size from the lower bounds."); - } - if (UpperBound != null && parameters.Count != UpperBound.Count) - { - throw new ArgumentException("The parameters can't have different size from the upper bounds."); - } - if (Scales != null && parameters.Count != Scales.Count) - { - throw new ArgumentException("The parameters can't have different size from the scales."); - } - if (IsFixed != null && parameters.Count != IsFixed.Count) - { - throw new ArgumentException("The parameters can't have different size from the IsFixed list."); - } - } - - private Matrix NumericalJacobian(Vector Pext, Vector currentValues, int accuracyOrder = 2) - { - const double sqrtEpsilon = 1.4901161193847656250E-8; // sqrt(machineEpsilon) - - Matrix derivertives = Matrix.Build.Dense(NumberOfObservations, NumberOfParameters); - - var d = 0.000003 * Pext.PointwiseAbs().PointwiseMaximum(sqrtEpsilon); - - var h = Vector.Build.Dense(NumberOfParameters); - for (int i = 0; i < NumberOfObservations; i++) - { - var x = ObservedX[i]; - for (int j = 0; j < NumberOfParameters; j++) - { - h[j] = d[j]; - - if (accuracyOrder >= 6) - { - // f'(x) = {- f(x - 3h) + 9f(x - 2h) - 45f(x - h) + 45f(x + h) - 9f(x + 2h) + f(x + 3h)} / 60h + O(h^6) - var f1 = userFunction(Pext - 3 * h, x); - var f2 = userFunction(Pext - 2 * h, x); - var f3 = userFunction(Pext - h, x); - var f4 = userFunction(Pext + h, x); - var f5 = userFunction(Pext + 2 * h, x); - var f6 = userFunction(Pext + 3 * h, x); - - var prime = (-f1 + 9 * f2 - 45 * f3 + 45 * f4 - 9 * f5 + f6) / (60 * h[j]); - derivertives[i, j] = prime; - } - else if (accuracyOrder == 5) - { - // f'(x) = {-137f(x) + 300f(x + h) - 300f(x + 2h) + 200f(x + 3h) - 75f(x + 4h) + 12f(x + 5h)} / 60h + O(h^5) - var f1 = currentValues[i]; - var f2 = userFunction(Pext + h, x); - var f3 = userFunction(Pext + 2 * h, x); - var f4 = userFunction(Pext + 3 * h, x); - var f5 = userFunction(Pext + 4 * h, x); - var f6 = userFunction(Pext + 5 * h, x); - - var prime = (-137 * f1 + 300 * f2 - 300 * f3 + 200 * f4 - 75 * f5 + 12 * f6) / (60 * h[j]); - derivertives[i, j] = prime; - } - else if (accuracyOrder == 4) - { - // f'(x) = {f(x - 2h) - 8f(x - h) + 8f(x + h) - f(x + 2h)} / 12h + O(h^4) - var f1 = userFunction(Pext - 2 * h, x); - var f2 = userFunction(Pext - h, x); - var f3 = userFunction(Pext + h, x); - var f4 = userFunction(Pext + 2 * h, x); - - var prime = (f1 - 8 * f2 + 8 * f3 - f4) / (12 * h[j]); - derivertives[i, j] = prime; - } - else if (accuracyOrder == 3) - { - // f'(x) = {-11f(x) + 18f(x + h) - 9f(x + 2h) + 2f(x + 3h)} / 6h + O(h^3) - var f1 = currentValues[i]; - var f2 = userFunction(Pext + h, x); - var f3 = userFunction(Pext + 2 * h, x); - var f4 = userFunction(Pext + 3 * h, x); - - var prime = (-11 * f1 + 18 * f2 - 9 * f3 + 2 * f4) / (6 * h[j]); - derivertives[i, j] = prime; - } - else if (accuracyOrder == 2) - { - // f'(x) = {f(x + h) - f(x - h)} / 2h + O(h^2) - var f1 = userFunction(Pext + h, x); - var f2 = userFunction(Pext - h, x); - - var prime = (f1 - f2) / (2 * h[j]); - derivertives[i, j] = prime; - } - else - { - // f'(x) = {- f(x) + f(x + h)} / h + O(h) - var f1 = currentValues[i]; - var f2 = userFunction(Pext + h, x); - - var prime = (-f1 + f2) / h[j]; - derivertives[i, j] = prime; - } - - h[j] = 0; - } - } - - return derivertives; - } - - private Vector ProjectParametersToInternal(Vector Pext) - { - var Pint = Pext.Clone(); - - if (LowerBound != null && UpperBound != null) - { - for (int i = 0; i < Pext.Count; i++) - { - Pint[i] = Math.Asin((2.0 * (Pext[i] - LowerBound[i]) / (UpperBound[i] - LowerBound[i])) - 1.0); - } - - return Pint; - } - else if (LowerBound != null && UpperBound == null) - { - for (int i = 0; i < Pext.Count; i++) - { - Pint[i] = (Scales == null) - ? Math.Sqrt(Math.Pow(Pext[i] - LowerBound[i] + 1.0, 2) - 1.0) - : Math.Sqrt(Math.Pow((Pext[i] - LowerBound[i]) / Scales[i] + 1.0, 2) - 1.0); - } - - return Pint; - } - else if (LowerBound == null && UpperBound != null) - { - for (int i = 0; i < Pext.Count; i++) - { - Pint[i] = (Scales == null) - ? Math.Sqrt(Math.Pow(UpperBound[i] - Pext[i] + 1.0, 2) - 1.0) - : Math.Sqrt(Math.Pow((UpperBound[i] - Pext[i]) / Scales[i] + 1.0, 2) - 1.0); - } - - return Pint; - } - else if (Scales != null) - { - for (int i = 0; i < Pext.Count; i++) - { - Pint[i] = Pext[i] / Scales[i]; - } - - return Pint; - } - - return Pint; - } - - private Vector ProjectParametersToExternal(Vector Pint) - { - var Pext = Pint.Clone(); - - if (LowerBound != null && UpperBound != null) - { - for (int i = 0; i < Pint.Count; i++) - { - Pext[i] = LowerBound[i] + (UpperBound[i] / 2.0 - LowerBound[i] / 2.0) * (Math.Sin(Pint[i]) + 1.0); - } - - return Pext; - } - else if (LowerBound != null && UpperBound == null) - { - for (int i = 0; i < Pint.Count; i++) - { - Pext[i] = (Scales == null) - ? LowerBound[i] + Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0 - : LowerBound[i] + Scales[i] * (Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0); - } - - return Pext; - } - else if (LowerBound == null && UpperBound != null) - { - for (int i = 0; i < Pint.Count; i++) - { - Pext[i] = (Scales == null) - ? UpperBound[i] - Math.Sqrt(Pint[i] * Pint[i] + 1.0) + 1.0 - : UpperBound[i] - Scales[i] * (Math.Sqrt(Pint[i] * Pint[i] + 1.0) - 1.0); - } - - return Pext; - } - else if (Scales != null) - { - for (int i = 0; i < Pint.Count; i++) - { - Pext[i] = Pint[i] * Scales[i]; - } - - return Pext; - } - - return Pext; - } - - private Vector ScaleFactorsOfJacobian(Vector Pint) - { - var scale = Vector.Build.Dense(Pint.Count, 1.0); - - if (LowerBound != null && UpperBound != null) - { - for (int i = 0; i < Pint.Count; i++) - { - scale[i] = (UpperBound[i] - LowerBound[i]) / 2.0 * Math.Cos(Pint[i]); - } - return scale; - } - else if (LowerBound != null && UpperBound == null) - { - for (int i = 0; i < Pint.Count; i++) - { - scale[i] = (Scales == null) - ? Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0) - : Scales[i] * Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); - } - return scale; - } - else if (LowerBound == null && UpperBound != null) - { - for (int i = 0; i < Pint.Count; i++) - { - scale[i] = (Scales == null) - ? -Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0) - : -Scales[i] * Pint[i] / Math.Sqrt(Pint[i] * Pint[i] + 1.0); - } - return scale; - } - else if (Scales != null) - { - return Scales; - } - - return scale; - } - - #endregion Private Methods - } -} diff --git a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs index d32c68f8..1ab8eb36 100644 --- a/src/Numerics/Optimization/TrustRegionMinimizerBase.cs +++ b/src/Numerics/Optimization/TrustRegionMinimizerBase.cs @@ -5,57 +5,32 @@ using System.Linq; namespace MathNet.Numerics.Optimization { - public abstract class TrustRegionMinimizerBase + public abstract class TrustRegionMinimizerBase : NonlinearMinimizerBase { - public static ITrustRegionSubproblem Subproblem; - - /// - /// The stopping threshold for infinity norm of the gradient. - /// - public static double GradientTolerance { get; set; } - /// - /// The stopping threshold for L2 norm of the change of the parameters. + /// The trust region subproblem. /// - public static double StepTolerance { get; set; } - - /// - /// The stopping threshold for the function value or L2 norm of the residuals. - /// - public static double FunctionTolerance { get; set; } + public static ITrustRegionSubproblem Subproblem; /// /// The stopping threshold for the trust region radius. /// public static double RadiusTolerance { get; set; } - /// - /// The maximum number of iterations. - /// - public int MaximumIterations { get; set; } - public TrustRegionMinimizerBase(ITrustRegionSubproblem subproblem, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-8, int maximumIterations = -1) + : base(gradientTolerance, stepTolerance, functionTolerance, maximumIterations) { if (subproblem == null) throw new ArgumentNullException("subproblem"); Subproblem = subproblem; - FunctionTolerance = functionTolerance; - GradientTolerance = gradientTolerance; - StepTolerance = stepTolerance; RadiusTolerance = radiusTolerance; - MaximumIterations = maximumIterations; } public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, Vector initialGuess, Vector lowerBound = null, Vector upperBound = null, Vector scales = null, List isFixed = null) { - if (objective == null) - throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - return Minimum(Subproblem, objective, initialGuess, lowerBound, upperBound, scales, isFixed, GradientTolerance, StepTolerance, FunctionTolerance, RadiusTolerance, MaximumIterations); } @@ -63,11 +38,6 @@ namespace MathNet.Numerics.Optimization public NonlinearMinimizationResult FindMinimum(IObjectiveModel objective, double[] initialGuess, double[] lowerBound = null, double[] upperBound = null, double[] scales = null, bool[] isFixed = null) { - if (objective == null) - throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - var lb = (lowerBound == null) ? null : CreateVector.Dense(lowerBound); var ub = (upperBound == null) ? null : CreateVector.Dense(upperBound); var sc = (scales == null) ? null : CreateVector.Dense(scales); @@ -133,23 +103,17 @@ namespace MathNet.Numerics.Optimization if (objective == null) throw new ArgumentNullException("objective"); - if (initialGuess == null) - throw new ArgumentNullException("initialGuess"); - objective.SetParameters(initialGuess, lowerBound, upperBound, scales, isFixed); + ValidateBounds(initialGuess, lowerBound, upperBound, scales); - ExitCondition exitCondition = ExitCondition.None; + objective.SetParameters(initialGuess, isFixed); - // Initialize objective - objective.FunctionEvaluations = 0; - objective.JacobianEvaluations = 0; - objective.IsFinished = false; + ExitCondition exitCondition = ExitCondition.None; // First, calculate function values and setup variables - objective.EvaluateAt(initialGuess); - var P = objective.Point; // current parameters - var RSS = objective.Value; // Residual Sum of Squares = R'R - var RSSinit = RSS; // RSS at initial gussing parameters + var P = ProjectToInternalParameters(initialGuess); // current internal parameters + var Pstep = Vector.Build.Dense(P.Count); // the change of parameters + var RSS = EvaluateFunction(objective, initialGuess); // Residual Sum of Squares if (maximumIterations < 0) { @@ -176,8 +140,9 @@ namespace MathNet.Numerics.Optimization } // evaluate projected gradient and Hessian - var Gradient = objective.Gradient; - var Hessian = objective.Hessian; + var jac = EvaluateJacobian(objective, P); + var Gradient = jac.Item1; // objective.Gradient; + var Hessian = jac.Item2; // objective.Hessian; // if ||g||_oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance) @@ -195,14 +160,16 @@ namespace MathNet.Numerics.Optimization delta = Math.Max(1.0, Math.Min(delta, maxDelta)); int iterations = 0; + bool hitBoundary = false; while (iterations < maximumIterations && exitCondition == ExitCondition.None) { iterations++; // solve the subproblem subproblem.Solve(objective, delta); - var Pstep = subproblem.Pstep; - var hitBoundary = subproblem.HitBoundary; + Pstep = subproblem.Pstep; + hitBoundary = subproblem.HitBoundary; + // predicted reduction = L(0) - L(Δp) = -Δp'g - 1/2 * Δp'HΔp var predictedReduction = -Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(Hessian * Pstep); @@ -213,10 +180,9 @@ namespace MathNet.Numerics.Optimization } var Pnew = P + Pstep; // parameters to test - - objective.EvaluateAt(Pnew); - var RSSnew = objective.Value; - + // evaluate function at Pnew + var RSSnew = EvaluateFunction(objective, Pnew); + // if RSS == NaN, stop if (double.IsNaN(RSSnew)) { @@ -238,7 +204,7 @@ namespace MathNet.Numerics.Optimization delta = delta * 0.25; if (delta <= radiusTolerance * (radiusTolerance + P.DotProduct(P))) { - exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters + exitCondition = ExitCondition.LackOfProgress; break; } } @@ -250,8 +216,9 @@ namespace MathNet.Numerics.Optimization RSS = RSSnew; // evaluate projected gradient and Hessian - Gradient = objective.Gradient; - Hessian = objective.Hessian; + jac = EvaluateJacobian(objective, P); + Gradient = jac.Item1; // objective.Gradient; + Hessian = jac.Item2; // objective.Hessian; // if ||g||_oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance)