Numerical Jacobian and Hessian wrappers

src/Numerics/Differentiation/NumericalHessian.cs

@@ -0,0 +1,119 @@
+// <copyright file="NumericalHessian.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2015 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Differentiation
+{
+    /// <summary>
+    /// Class for evaluating the Hessian of a smooth continuously differentiable function using finite differences. 
+    /// By default, a central 3-point method is used.
+    /// </summary>
+    public class NumericalHessian
+    {
+        /// <summary>
+        /// Number of function evaluations.
+        /// </summary>
+        public int FunctionEvaluations
+        {
+            get { return _df.Evaluations; }
+        }
+
+        private readonly NumericalDerivative _df;
+
+        /// <summary>
+        /// Creates a numerical Hessian object with a three point central difference method.
+        /// </summary>
+        public NumericalHessian() : this(3, 1) { }
+
+        /// <summary>
+        /// Creates a numerical Hessian with a specified differentiation scheme.
+        /// </summary>
+        /// <param name="points">Number of points for Hessian evaluation.</param>
+        /// <param name="center">Center point for differentiation.</param>
+        public NumericalHessian(int points, int center)
+        {
+            _df = new NumericalDerivative(points, center);
+        }
+
+        /// <summary>
+        /// Evaluates the Hessian of the scalar univariate function f at points x.
+        /// </summary>
+        /// <param name="f">Scalar univariate function handle.</param>
+        /// <param name="x">Point at which to evaluate Hessian.</param>
+        /// <returns>Hessian tensor.</returns>
+        public double[] Evaluate(Func<double, double> f, double x)
+        {
+            return new[] { _df.EvaluateDerivative(f, x, 2) };
+        }
+
+        /// <summary>
+        /// Evaluates the Hessian of a multivariate function f at points x.
+        /// </summary>
+        /// <remarks>
+        /// This method of computing the Hessian is only vaid for Lipschitz continuous functions. 
+        /// The function mirrors the Hessian along the diagonal since d2f/dxdy = d2f/dydx for continuously differentiable functions. 
+        /// </remarks>
+        /// <param name="f">Multivariate function handle.></param>
+        /// <param name="x">Points at which to evaluate Hessian.></param>
+        /// <returns>Hessian tensor.</returns>
+        public double[,] Evaluate(Func<double[], double> f, double[] x)
+        {
+            var hessian = new double[x.Length, x.Length];
+
+            // Compute diagonal elements
+            for (var row = 0; row < x.Length; row++)
+            {
+                hessian[row, row] = _df.EvaluatePartialDerivative(f, x, row, 2);
+            }
+
+            // Compute non-diagonal elements
+            for (var row = 0; row < x.Length; row++)
+            {
+                for (var col = 0; col < row; col++)
+                {
+                    var mixedPartial = _df.EvaluateMixedPartialDerivative(f, x, new[] { row, col }, 2);
+
+                    hessian[row, col] = mixedPartial;
+                    hessian[col, row] = mixedPartial;
+                }
+            }
+            return hessian;
+        }
+
+        /// <summary>
+        /// Resets the function evaluation counter for the Hessian.
+        /// </summary>
+        public void ResetFunctionEvaluations()
+        {
+            _df.ResetEvaluations();
+        }
+    }
+}

src/Numerics/Differentiation/NumericalJacobian.cs

@@ -0,0 +1,172 @@
+// <copyright file="NumericalJacobian.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+//
+// Copyright (c) 2009-2015 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+
+namespace MathNet.Numerics.Differentiation
+{
+    /// <summary>
+    /// Class for evaluating the Jacobian of a function using finite differences. 
+    /// By default, a central 3-point method is used.
+    /// </summary>
+    public class NumericalJacobian
+    {
+        /// <summary>
+        /// Number of function evaluations.
+        /// </summary>
+        public int FunctionEvaluations
+        {
+            get { return _df.Evaluations; }
+        }
+
+        private readonly NumericalDerivative _df;
+
+        /// <summary>
+        /// Creates a numerical Jacobian object with a three point central difference method.
+        /// </summary>
+        public NumericalJacobian() : this(3, 1) { }
+
+        /// <summary>
+        /// Creates a numerical Jacobian with a specified differentiation scheme.
+        /// </summary>
+        /// <param name="points">Number of points for Jacobian evaluation.</param>
+        /// <param name="center">Center point for differentation.</param>
+        public NumericalJacobian(int points, int center)
+        {
+            _df = new NumericalDerivative(points, center);
+        }
+
+        /// <summary>
+        /// Evaluates the Jacbian of scalar univariate function f at point x.
+        /// </summary>
+        /// <param name="f">Scalar univariate function handle.</param>
+        /// <param name="x">Point at which to evaluate Jacobian.</param>
+        /// <returns>Jacobian vector.</returns>
+        public double[] Evaluate(Func<double, double> f, double x)
+        {
+            return new[] { _df.EvaluateDerivative(f, x, 1) };
+        }
+
+        /// <summary>
+        /// Evaluates the Jacobian of a multivariate function f at vector x.
+        /// </summary>
+        /// <remarks>
+        /// This function assumes that the length of vector x consistent with the argument count of f.
+        /// </remarks>
+        /// <param name="f">Multivariate function handle.</param>
+        /// <param name="x">Points at which to evaluate Jacobian.</param>
+        /// <returns>Jacobian vector.</returns>
+        public double[] Evaluate(Func<double[], double> f, double[] x)
+        {
+            var jacobian = new double[x.Length];
+
+            for (var i = 0; i < jacobian.Length; i++)
+                jacobian[i] = _df.EvaluatePartialDerivative(f, x, i, 1);
+
+            return jacobian;
+        }
+
+        /// <summary>
+        /// Evaluates the Jacobian of a multivariate function f at vector x given a current function value.
+        /// </summary>
+        /// <remarks>
+        /// To minimize the number of function evaluations, a user can supply the current value of the function 
+        /// to be used in computing the Jacobian. This value must correspond to the "center" location for the 
+        /// finite differencing. If a scheme is used where the center value is not evaluated, this will provide no 
+        /// added efficiency. This method also assumes that the length of vector x consistent with the argument count of f.
+        /// </remarks>
+        /// <param name="f">Multivariate function handle.</param>
+        /// <param name="x">Points at which to evaluate Jacobian.</param>
+        /// <param name="currentValue">Current function value at finite difference center.</param>
+        /// <returns>Jacobian vector.</returns>
+        public double[] Evaluate(Func<double[], double> f, double[] x, double currentValue)
+        {
+            var jacobian = new double[x.Length];
+
+            for (var i = 0; i < jacobian.Length; i++)
+                jacobian[i] = _df.EvaluatePartialDerivative(f, x, i, 1, currentValue);
+
+            return jacobian;
+        }
+
+        /// <summary>
+        /// Evaluates the Jacobian of a multivariate function array f at vector x.
+        /// </summary>
+        /// <param name="f">Multivariate function array handle.</param>
+        /// <param name="x">Vector at which to evaluate Jacobian.</param>
+        /// <returns>Jacobian matrix.</returns>
+        public double[,] Evaluate(Func<double[], double>[] f, double[] x)
+        {
+            var jacobian = new double[f.Length, x.Length];
+            for (int i = 0; i < f.Length; i++)
+            {
+                var gradient = Evaluate(f[i], x);
+                for (int j = 0; j < gradient.Length; j++)
+                    jacobian[i, j] = gradient[j];
+            }
+
+            return jacobian;
+        }
+
+        /// <summary>
+        /// Evaluates the Jacobian of a multivariate function array f at vector x given a vector of current function values.
+        /// </summary>
+        /// <remarks>
+        /// To minimize the number of function evaluations, a user can supply a vector of current values of the functions 
+        /// to be used in computing the Jacobian. These value must correspond to the "center" location for the 
+        /// finite differencing. If a scheme is used where the center value is not evaluated, this will provide no 
+        /// added efficiency. This method also assumes that the length of vector x consistent with the argument count of f.
+        /// </remarks>
+        /// <param name="f">Multivariate function array handle.</param>
+        /// <param name="x">Vector at which to evaluate Jacobian.</param>
+        /// <param name="currentValues">Vector of current function values.</param>
+        /// <returns>Jacobian matrix.</returns>
+        public double[,] Evaluate(Func<double[], double>[] f, double[] x, double[] currentValues)
+        {
+            var jacobian = new double[f.Length, x.Length];
+            for (int i = 0; i < f.Length; i++)
+            {
+                var gradient = Evaluate(f[i], x, currentValues[i]);
+                for (int j = 0; j < gradient.Length; j++)
+                    jacobian[i, j] = gradient[j];
+            }
+
+            return jacobian;
+        }
+
+        /// <summary>
+        /// Resets the function evaluation counter for the Jacobian.
+        /// </summary>
+        public void ResetFunctionEvaluations()
+        {
+            _df.ResetEvaluations();
+        }
+    }
+}

src/Numerics/Numerics.csproj

@@ -82,6 +82,8 @@
   <ItemGroup>
     <Compile Include="Differentiation\FiniteDifferenceCoefficients.cs" />
     <Compile Include="Differentiation\NumericalDerivative.cs" />
+    <Compile Include="Differentiation\NumericalHessian.cs" />
+    <Compile Include="Differentiation\NumericalJacobian.cs" />
     <Compile Include="Distributions\Triangular.cs" />
     <Compile Include="Euclid.cs" />
     <Compile Include="Generate.cs" />

src/UnitTests/DifferentiationTests/NumericalHessianTests.cs

@@ -0,0 +1,76 @@
+// <copyright file="NumericalHessianTests.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+// Copyright (c) 2009-2015 Math.NET
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+namespace MathNet.Numerics.UnitTests.DifferentiationTests
+{
+    using System;
+    using Differentiation;
+    using NUnit.Framework;
+
+    [TestFixture, Category("Differentiation")]
+    class NumericalHessianTests
+    {
+        [Test]
+        public void ScalarFunctonSecondDerivativeTest()
+        {
+            // d2/dx2((2x)^2) = 8
+            Func<double, double> f = x => Math.Pow(2 * x, 2);
+            var J = new NumericalHessian();
+            var jeval = J.Evaluate(f, 2);
+            Assert.AreEqual((double)8, jeval[0], 1e-7);
+        }
+
+        [Test]
+        public void OneEquationTwoVarVectorFunctionTest()
+        {
+            Func<double[], double> f = x => 4 * Math.Pow(x[0], 2) + 3 * Math.Pow(x[1], 3);
+            var H = new NumericalHessian();
+            var heval = H.Evaluate(f, new double[] { 1, 1 });
+            var solution = new double[,] { { 8, 0 }, { 0, 18 } };
+            Assert.AreEqual(solution, heval);
+            Assert.AreEqual(12, H.FunctionEvaluations);
+            H.ResetFunctionEvaluations();
+            Assert.AreEqual(0, H.FunctionEvaluations);
+        }
+
+        [Test]
+        public void RosenbrockFunctionHessianTest()
+        {
+            Func<double[], double> f = x => Math.Pow((1 - x[0]), 2) + 100 * Math.Pow(x[1] - Math.Pow(x[0], 2), 2);
+            var H = new NumericalHessian(5, 2);
+            var heval = H.Evaluate(f, new double[] { 2, 3 });
+            var solution = new double[,] { { 3602, -800 }, { -800, 200 } };
+            for (int row = 0; row < solution.Rank; row++)
+            {
+                for (int col = 0; col < solution.Rank; col++)
+                {
+                    Assert.AreEqual(solution[row, col], heval[row, col], 1e-7);
+                }
+            }
+
+        }
+    }
+}

src/UnitTests/DifferentiationTests/NumericalJacobianTests.cs

@@ -0,0 +1,75 @@
+// <copyright file="NumericalJacobianTests.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+// http://mathnetnumerics.codeplex.com
+// Copyright (c) 2009-2015 Math.NET
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+namespace MathNet.Numerics.UnitTests.DifferentiationTests
+{
+    using System;
+    using Differentiation;
+    using NUnit.Framework;
+
+    [TestFixture, Category("Differentiation")]
+    class NumericalJacobianTests
+    {
+        [Test]
+        public void OneEquationVectorFunctionTest()
+        {
+            Func<double[], double> f = x => Math.Sin(x[0] * x[1]) * Math.Exp(-x[0] / 2) + x[1] / x[0];
+            var J = new NumericalJacobian();
+            var jeval = J.Evaluate(f, new double[] { 1, 1 });
+            Assert.AreEqual(-0.92747906175, jeval[0], 1e-9);
+            Assert.AreEqual(1.32770991402, jeval[1], 1e-9);
+            Assert.AreEqual(4, J.FunctionEvaluations);
+            J.ResetFunctionEvaluations();
+            Assert.AreEqual(0, J.FunctionEvaluations);
+        }
+
+        [Test]
+        public void ScalarFunctonDerivativeTest()
+        {
+            Func<double, double> f = x => 1 / x;
+            var J = new NumericalJacobian();
+            var jeval = J.Evaluate(f, 2);
+            Assert.AreEqual((double)-1 / 4, jeval[0], 1e-7);
+        }
+
+        [Test]
+        public void VectorFunctionJacobianTest()
+        {
+            Func<double[], double>[] f = 
+            {
+                (x) => Math.Pow(x[0], 2)*x[1], 
+                (x) => 5*x[0] + Math.Sin(x[1])
+            };
+            var j = new NumericalJacobian();
+            var jeval = j.Evaluate(f, new double[] { 1, 1 });
+            Assert.AreEqual(2, jeval[0, 0]);
+            Assert.AreEqual(1, jeval[0, 1]);
+            Assert.AreEqual(5, jeval[1, 0]);
+            Assert.AreEqual(Math.Cos(1), jeval[1, 1], 1e-7);
+
+        }
+    }
+}

src/UnitTests/UnitTests.csproj

@@ -79,6 +79,8 @@
     <Compile Include="ComplexTests\ComplexTest.TextHandling.cs" />
     <Compile Include="DifferentiationTests\FiniteDifferenceCoefficientsTests.cs" />
     <Compile Include="DifferentiationTests\NumericalDerivativeTests.cs" />
+    <Compile Include="DifferentiationTests\NumericalHessianTests.cs" />
+    <Compile Include="DifferentiationTests\NumericalJacobianTests.cs" />
     <Compile Include="DistanceTests.cs" />
     <Compile Include="DistributionTests\CommonDistributionTests.cs" />
     <Compile Include="DistributionTests\Continuous\BetaTests.cs" />