373 lines
15 KiB
C++
373 lines
15 KiB
C++
|
// Ceres Solver - A fast non-linear least squares minimizer
|
||
|
|
// Copyright 2015 Google Inc. All rights reserved.
|
||
|
|
// http://ceres-solver.org/
|
||
|
|
//
|
||
|
|
// Redistribution and use in source and binary forms, with or without
|
||
|
|
// modification, are permitted provided that the following conditions are met:
|
||
|
|
//
|
||
|
|
// * Redistributions of source code must retain the above copyright notice,
|
||
|
|
// this list of conditions and the following disclaimer.
|
||
|
|
// * Redistributions in binary form must reproduce the above copyright notice,
|
||
|
|
// this list of conditions and the following disclaimer in the documentation
|
||
|
|
// and/or other materials provided with the distribution.
|
||
|
|
// * Neither the name of Google Inc. nor the names of its contributors may be
|
||
|
|
// used to endorse or promote products derived from this software without
|
||
|
|
// specific prior written permission.
|
||
|
|
//
|
||
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||
|
|
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||
|
|
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||
|
|
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||
|
|
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||
|
|
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||
|
|
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||
|
|
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||
|
|
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||
|
|
// POSSIBILITY OF SUCH DAMAGE.
|
||
|
|
//
|
||
|
|
// Author: sameeragarwal@google.com (Sameer Agarwal)
|
||
|
|
|
||
|
|
#include "ceres/line_search_direction.h"
|
||
|
|
#include "ceres/line_search_minimizer.h"
|
||
|
|
#include "ceres/low_rank_inverse_hessian.h"
|
||
|
|
#include "ceres/internal/eigen.h"
|
||
|
|
#include "glog/logging.h"
|
||
|
|
|
||
|
|
namespace ceres {
|
||
|
|
namespace internal {
|
||
|
|
|
||
|
|
class SteepestDescent : public LineSearchDirection {
|
||
|
|
public:
|
||
|
|
virtual ~SteepestDescent() {}
|
||
|
|
bool NextDirection(const LineSearchMinimizer::State& previous,
|
||
|
|
const LineSearchMinimizer::State& current,
|
||
|
|
Vector* search_direction) {
|
||
|
|
*search_direction = -current.gradient;
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
class NonlinearConjugateGradient : public LineSearchDirection {
|
||
|
|
public:
|
||
|
|
NonlinearConjugateGradient(const NonlinearConjugateGradientType type,
|
||
|
|
const double function_tolerance)
|
||
|
|
: type_(type),
|
||
|
|
function_tolerance_(function_tolerance) {
|
||
|
|
}
|
||
|
|
|
||
|
|
bool NextDirection(const LineSearchMinimizer::State& previous,
|
||
|
|
const LineSearchMinimizer::State& current,
|
||
|
|
Vector* search_direction) {
|
||
|
|
double beta = 0.0;
|
||
|
|
Vector gradient_change;
|
||
|
|
switch (type_) {
|
||
|
|
case FLETCHER_REEVES:
|
||
|
|
beta = current.gradient_squared_norm / previous.gradient_squared_norm;
|
||
|
|
break;
|
||
|
|
case POLAK_RIBIERE:
|
||
|
|
gradient_change = current.gradient - previous.gradient;
|
||
|
|
beta = (current.gradient.dot(gradient_change) /
|
||
|
|
previous.gradient_squared_norm);
|
||
|
|
break;
|
||
|
|
case HESTENES_STIEFEL:
|
||
|
|
gradient_change = current.gradient - previous.gradient;
|
||
|
|
beta = (current.gradient.dot(gradient_change) /
|
||
|
|
previous.search_direction.dot(gradient_change));
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
LOG(FATAL) << "Unknown nonlinear conjugate gradient type: " << type_;
|
||
|
|
}
|
||
|
|
|
||
|
|
*search_direction = -current.gradient + beta * previous.search_direction;
|
||
|
|
const double directional_derivative =
|
||
|
|
current.gradient.dot(*search_direction);
|
||
|
|
if (directional_derivative > -function_tolerance_) {
|
||
|
|
LOG(WARNING) << "Restarting non-linear conjugate gradients: "
|
||
|
|
<< directional_derivative;
|
||
|
|
*search_direction = -current.gradient;
|
||
|
|
}
|
||
|
|
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
const NonlinearConjugateGradientType type_;
|
||
|
|
const double function_tolerance_;
|
||
|
|
};
|
||
|
|
|
||
|
|
class LBFGS : public LineSearchDirection {
|
||
|
|
public:
|
||
|
|
LBFGS(const int num_parameters,
|
||
|
|
const int max_lbfgs_rank,
|
||
|
|
const bool use_approximate_eigenvalue_bfgs_scaling)
|
||
|
|
: low_rank_inverse_hessian_(num_parameters,
|
||
|
|
max_lbfgs_rank,
|
||
|
|
use_approximate_eigenvalue_bfgs_scaling),
|
||
|
|
is_positive_definite_(true) {}
|
||
|
|
|
||
|
|
virtual ~LBFGS() {}
|
||
|
|
|
||
|
|
bool NextDirection(const LineSearchMinimizer::State& previous,
|
||
|
|
const LineSearchMinimizer::State& current,
|
||
|
|
Vector* search_direction) {
|
||
|
|
CHECK(is_positive_definite_)
|
||
|
|
<< "Ceres bug: NextDirection() called on L-BFGS after inverse Hessian "
|
||
|
|
<< "approximation has become indefinite, please contact the "
|
||
|
|
<< "developers!";
|
||
|
|
|
||
|
|
low_rank_inverse_hessian_.Update(
|
||
|
|
previous.search_direction * previous.step_size,
|
||
|
|
current.gradient - previous.gradient);
|
||
|
|
|
||
|
|
search_direction->setZero();
|
||
|
|
low_rank_inverse_hessian_.RightMultiply(current.gradient.data(),
|
||
|
|
search_direction->data());
|
||
|
|
*search_direction *= -1.0;
|
||
|
|
|
||
|
|
if (search_direction->dot(current.gradient) >= 0.0) {
|
||
|
|
LOG(WARNING) << "Numerical failure in L-BFGS update: inverse Hessian "
|
||
|
|
<< "approximation is not positive definite, and thus "
|
||
|
|
<< "initial gradient for search direction is positive: "
|
||
|
|
<< search_direction->dot(current.gradient);
|
||
|
|
is_positive_definite_ = false;
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
LowRankInverseHessian low_rank_inverse_hessian_;
|
||
|
|
bool is_positive_definite_;
|
||
|
|
};
|
||
|
|
|
||
|
|
class BFGS : public LineSearchDirection {
|
||
|
|
public:
|
||
|
|
BFGS(const int num_parameters,
|
||
|
|
const bool use_approximate_eigenvalue_scaling)
|
||
|
|
: num_parameters_(num_parameters),
|
||
|
|
use_approximate_eigenvalue_scaling_(use_approximate_eigenvalue_scaling),
|
||
|
|
initialized_(false),
|
||
|
|
is_positive_definite_(true) {
|
||
|
|
LOG_IF(WARNING, num_parameters_ >= 1e3)
|
||
|
|
<< "BFGS line search being created with: " << num_parameters_
|
||
|
|
<< " parameters, this will allocate a dense approximate inverse Hessian"
|
||
|
|
<< " of size: " << num_parameters_ << " x " << num_parameters_
|
||
|
|
<< ", consider using the L-BFGS memory-efficient line search direction "
|
||
|
|
<< "instead.";
|
||
|
|
// Construct inverse_hessian_ after logging warning about size s.t. if the
|
||
|
|
// allocation crashes us, the log will highlight what the issue likely was.
|
||
|
|
inverse_hessian_ = Matrix::Identity(num_parameters, num_parameters);
|
||
|
|
}
|
||
|
|
|
||
|
|
virtual ~BFGS() {}
|
||
|
|
|
||
|
|
bool NextDirection(const LineSearchMinimizer::State& previous,
|
||
|
|
const LineSearchMinimizer::State& current,
|
||
|
|
Vector* search_direction) {
|
||
|
|
CHECK(is_positive_definite_)
|
||
|
|
<< "Ceres bug: NextDirection() called on BFGS after inverse Hessian "
|
||
|
|
<< "approximation has become indefinite, please contact the "
|
||
|
|
<< "developers!";
|
||
|
|
|
||
|
|
const Vector delta_x = previous.search_direction * previous.step_size;
|
||
|
|
const Vector delta_gradient = current.gradient - previous.gradient;
|
||
|
|
const double delta_x_dot_delta_gradient = delta_x.dot(delta_gradient);
|
||
|
|
|
||
|
|
// The (L)BFGS algorithm explicitly requires that the secant equation:
|
||
|
|
//
|
||
|
|
// B_{k+1} * s_k = y_k
|
||
|
|
//
|
||
|
|
// Is satisfied at each iteration, where B_{k+1} is the approximated
|
||
|
|
// Hessian at the k+1-th iteration, s_k = (x_{k+1} - x_{k}) and
|
||
|
|
// y_k = (grad_{k+1} - grad_{k}). As the approximated Hessian must be
|
||
|
|
// positive definite, this is equivalent to the condition:
|
||
|
|
//
|
||
|
|
// s_k^T * y_k > 0 [s_k^T * B_{k+1} * s_k = s_k^T * y_k > 0]
|
||
|
|
//
|
||
|
|
// This condition would always be satisfied if the function was strictly
|
||
|
|
// convex, alternatively, it is always satisfied provided that a Wolfe line
|
||
|
|
// search is used (even if the function is not strictly convex). See [1]
|
||
|
|
// (p138) for a proof.
|
||
|
|
//
|
||
|
|
// Although Ceres will always use a Wolfe line search when using (L)BFGS,
|
||
|
|
// practical implementation considerations mean that the line search
|
||
|
|
// may return a point that satisfies only the Armijo condition, and thus
|
||
|
|
// could violate the Secant equation. As such, we will only use a step
|
||
|
|
// to update the Hessian approximation if:
|
||
|
|
//
|
||
|
|
// s_k^T * y_k > tolerance
|
||
|
|
//
|
||
|
|
// It is important that tolerance is very small (and >=0), as otherwise we
|
||
|
|
// might skip the update too often and fail to capture important curvature
|
||
|
|
// information in the Hessian. For example going from 1e-10 -> 1e-14
|
||
|
|
// improves the NIST benchmark score from 43/54 to 53/54.
|
||
|
|
//
|
||
|
|
// [1] Nocedal J, Wright S, Numerical Optimization, 2nd Ed. Springer, 1999.
|
||
|
|
//
|
||
|
|
// TODO(alexs.mac): Consider using Damped BFGS update instead of
|
||
|
|
// skipping update.
|
||
|
|
const double kBFGSSecantConditionHessianUpdateTolerance = 1e-14;
|
||
|
|
if (delta_x_dot_delta_gradient <=
|
||
|
|
kBFGSSecantConditionHessianUpdateTolerance) {
|
||
|
|
VLOG(2) << "Skipping BFGS Update, delta_x_dot_delta_gradient too "
|
||
|
|
<< "small: " << delta_x_dot_delta_gradient << ", tolerance: "
|
||
|
|
<< kBFGSSecantConditionHessianUpdateTolerance
|
||
|
|
<< " (Secant condition).";
|
||
|
|
} else {
|
||
|
|
// Update dense inverse Hessian approximation.
|
||
|
|
|
||
|
|
if (!initialized_ && use_approximate_eigenvalue_scaling_) {
|
||
|
|
// Rescale the initial inverse Hessian approximation (H_0) to be
|
||
|
|
// iteratively updated so that it is of similar 'size' to the true
|
||
|
|
// inverse Hessian at the start point. As shown in [1]:
|
||
|
|
//
|
||
|
|
// \gamma = (delta_gradient_{0}' * delta_x_{0}) /
|
||
|
|
// (delta_gradient_{0}' * delta_gradient_{0})
|
||
|
|
//
|
||
|
|
// Satisfies:
|
||
|
|
//
|
||
|
|
// (1 / \lambda_m) <= \gamma <= (1 / \lambda_1)
|
||
|
|
//
|
||
|
|
// Where \lambda_1 & \lambda_m are the smallest and largest eigenvalues
|
||
|
|
// of the true initial Hessian (not the inverse) respectively. Thus,
|
||
|
|
// \gamma is an approximate eigenvalue of the true inverse Hessian, and
|
||
|
|
// choosing: H_0 = I * \gamma will yield a starting point that has a
|
||
|
|
// similar scale to the true inverse Hessian. This technique is widely
|
||
|
|
// reported to often improve convergence, however this is not
|
||
|
|
// universally true, particularly if there are errors in the initial
|
||
|
|
// gradients, or if there are significant differences in the sensitivity
|
||
|
|
// of the problem to the parameters (i.e. the range of the magnitudes of
|
||
|
|
// the components of the gradient is large).
|
||
|
|
//
|
||
|
|
// The original origin of this rescaling trick is somewhat unclear, the
|
||
|
|
// earliest reference appears to be Oren [1], however it is widely
|
||
|
|
// discussed without specific attributation in various texts including
|
||
|
|
// [2] (p143).
|
||
|
|
//
|
||
|
|
// [1] Oren S.S., Self-scaling variable metric (SSVM) algorithms
|
||
|
|
// Part II: Implementation and experiments, Management Science,
|
||
|
|
// 20(5), 863-874, 1974.
|
||
|
|
// [2] Nocedal J., Wright S., Numerical Optimization, Springer, 1999.
|
||
|
|
const double approximate_eigenvalue_scale =
|
||
|
|
delta_x_dot_delta_gradient / delta_gradient.dot(delta_gradient);
|
||
|
|
inverse_hessian_ *= approximate_eigenvalue_scale;
|
||
|
|
|
||
|
|
VLOG(4) << "Applying approximate_eigenvalue_scale: "
|
||
|
|
<< approximate_eigenvalue_scale << " to initial inverse "
|
||
|
|
<< "Hessian approximation.";
|
||
|
|
}
|
||
|
|
initialized_ = true;
|
||
|
|
|
||
|
|
// Efficient O(num_parameters^2) BFGS update [2].
|
||
|
|
//
|
||
|
|
// Starting from dense BFGS update detailed in Nocedal [2] p140/177 and
|
||
|
|
// using: y_k = delta_gradient, s_k = delta_x:
|
||
|
|
//
|
||
|
|
// \rho_k = 1.0 / (s_k' * y_k)
|
||
|
|
// V_k = I - \rho_k * y_k * s_k'
|
||
|
|
// H_k = (V_k' * H_{k-1} * V_k) + (\rho_k * s_k * s_k')
|
||
|
|
//
|
||
|
|
// This update involves matrix, matrix products which naively O(N^3),
|
||
|
|
// however we can exploit our knowledge that H_k is positive definite
|
||
|
|
// and thus by defn. symmetric to reduce the cost of the update:
|
||
|
|
//
|
||
|
|
// Expanding the update above yields:
|
||
|
|
//
|
||
|
|
// H_k = H_{k-1} +
|
||
|
|
// \rho_k * ( (1.0 + \rho_k * y_k' * H_k * y_k) * s_k * s_k' -
|
||
|
|
// (s_k * y_k' * H_k + H_k * y_k * s_k') )
|
||
|
|
//
|
||
|
|
// Using: A = (s_k * y_k' * H_k), and the knowledge that H_k = H_k', the
|
||
|
|
// last term simplifies to (A + A'). Note that although A is not symmetric
|
||
|
|
// (A + A') is symmetric. For ease of construction we also define
|
||
|
|
// B = (1 + \rho_k * y_k' * H_k * y_k) * s_k * s_k', which is by defn
|
||
|
|
// symmetric due to construction from: s_k * s_k'.
|
||
|
|
//
|
||
|
|
// Now we can write the BFGS update as:
|
||
|
|
//
|
||
|
|
// H_k = H_{k-1} + \rho_k * (B - (A + A'))
|
||
|
|
|
||
|
|
// For efficiency, as H_k is by defn. symmetric, we will only maintain the
|
||
|
|
// *lower* triangle of H_k (and all intermediary terms).
|
||
|
|
|
||
|
|
const double rho_k = 1.0 / delta_x_dot_delta_gradient;
|
||
|
|
|
||
|
|
// Calculate: A = s_k * y_k' * H_k
|
||
|
|
Matrix A = delta_x * (delta_gradient.transpose() *
|
||
|
|
inverse_hessian_.selfadjointView<Eigen::Lower>());
|
||
|
|
|
||
|
|
// Calculate scalar: (1 + \rho_k * y_k' * H_k * y_k)
|
||
|
|
const double delta_x_times_delta_x_transpose_scale_factor =
|
||
|
|
(1.0 + (rho_k * delta_gradient.transpose() *
|
||
|
|
inverse_hessian_.selfadjointView<Eigen::Lower>() *
|
||
|
|
delta_gradient));
|
||
|
|
// Calculate: B = (1 + \rho_k * y_k' * H_k * y_k) * s_k * s_k'
|
||
|
|
Matrix B = Matrix::Zero(num_parameters_, num_parameters_);
|
||
|
|
B.selfadjointView<Eigen::Lower>().
|
||
|
|
rankUpdate(delta_x, delta_x_times_delta_x_transpose_scale_factor);
|
||
|
|
|
||
|
|
// Finally, update inverse Hessian approximation according to:
|
||
|
|
// H_k = H_{k-1} + \rho_k * (B - (A + A')). Note that (A + A') is
|
||
|
|
// symmetric, even though A is not.
|
||
|
|
inverse_hessian_.triangularView<Eigen::Lower>() +=
|
||
|
|
rho_k * (B - A - A.transpose());
|
||
|
|
}
|
||
|
|
|
||
|
|
*search_direction =
|
||
|
|
inverse_hessian_.selfadjointView<Eigen::Lower>() *
|
||
|
|
(-1.0 * current.gradient);
|
||
|
|
|
||
|
|
if (search_direction->dot(current.gradient) >= 0.0) {
|
||
|
|
LOG(WARNING) << "Numerical failure in BFGS update: inverse Hessian "
|
||
|
|
<< "approximation is not positive definite, and thus "
|
||
|
|
<< "initial gradient for search direction is positive: "
|
||
|
|
<< search_direction->dot(current.gradient);
|
||
|
|
is_positive_definite_ = false;
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
const int num_parameters_;
|
||
|
|
const bool use_approximate_eigenvalue_scaling_;
|
||
|
|
Matrix inverse_hessian_;
|
||
|
|
bool initialized_;
|
||
|
|
bool is_positive_definite_;
|
||
|
|
};
|
||
|
|
|
||
|
|
LineSearchDirection*
|
||
|
|
LineSearchDirection::Create(const LineSearchDirection::Options& options) {
|
||
|
|
if (options.type == STEEPEST_DESCENT) {
|
||
|
|
return new SteepestDescent;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (options.type == NONLINEAR_CONJUGATE_GRADIENT) {
|
||
|
|
return new NonlinearConjugateGradient(
|
||
|
|
options.nonlinear_conjugate_gradient_type,
|
||
|
|
options.function_tolerance);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (options.type == ceres::LBFGS) {
|
||
|
|
return new ceres::internal::LBFGS(
|
||
|
|
options.num_parameters,
|
||
|
|
options.max_lbfgs_rank,
|
||
|
|
options.use_approximate_eigenvalue_bfgs_scaling);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (options.type == ceres::BFGS) {
|
||
|
|
return new ceres::internal::BFGS(
|
||
|
|
options.num_parameters,
|
||
|
|
options.use_approximate_eigenvalue_bfgs_scaling);
|
||
|
|
}
|
||
|
|
|
||
|
|
LOG(ERROR) << "Unknown line search direction type: " << options.type;
|
||
|
|
return NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace internal
|
||
|
|
} // namespace ceres
|