Back
Quasi-Newton Methods: A New Direction
Four decades after their invention, quasi-Newton methods are still state of the art in unconstrained numerical optimization. Although not usually interpreted thus, these are learning algorithms that fit a local quadratic approximation to the objective function. We show that many, including the most popular, quasi-Newton methods can be interpreted as approximations of Bayesian linear regression under varying prior assumptions. This new notion elucidates some shortcomings of classical algorithms, and lights the way to a novel nonparametric quasi-Newton method, which is able to make more efficient use of available information at computational cost similar to its predecessors.
@article{hennig13, title = {Quasi-Newton Methods: A New Direction}, journal = {Journal of Machine Learning Research}, abstract = {Four decades after their invention, quasi-Newton methods are still state of the art in unconstrained numerical optimization. Although not usually interpreted thus, these are learning algorithms that fit a local quadratic approximation to the objective function. We show that many, including the most popular, quasi-Newton methods can be interpreted as approximations of Bayesian linear regression under varying prior assumptions. This new notion elucidates some shortcomings of classical algorithms, and lights the way to a novel nonparametric quasi-Newton method, which is able to make more efficient use of available information at computational cost similar to its predecessors.}, volume = {14}, number = {1}, pages = {843--865}, month = mar, year = {2013}, slug = {hennig13}, author = {Hennig, Philipp and Kiefel, Martin}, url = {http://www.jmlr.org/papers/volume14/hennig13a/hennig13a.pdf}, month_numeric = {3} }