Quasi-Newton Methods: A New Direction
Four decades after their invention, quasi-Newton methods are still state of the art in unconstrained numerical optimization. Although not usually interpreted thus, these are learning algorithms that fit a local quadratic approximation to the objective function. We show that many, including the most popular, quasi-Newton methods can be interpreted as approximations of Bayesian linear regression under varying prior assumptions. This new notion elucidates some shortcomings of classical algorithms, and lights the way to a novel nonparametric quasi-Newton method, which is able to make more efficient use of available information at computational cost similar to its predecessors.
| Author(s): | Hennig, Philipp and Kiefel, Martin |
| Journal: | Journal of Machine Learning Research |
| Volume: | 14 |
| Number (issue): | 1 |
| Pages: | 843--865 |
| Year: | 2013 |
| Month: | March |
| Project(s): | |
| Bibtex Type: | Article (article) |
| URL: | http://www.jmlr.org/papers/volume14/hennig13a/hennig13a.pdf |
| Electronic Archiving: | grant_archive |
| Links: | |
| Attachments: | |
BibTex
@article{hennig13,
title = {Quasi-Newton Methods: A New Direction},
journal = {Journal of Machine Learning Research},
abstract = {Four decades after their invention, quasi-Newton methods are still state of the art in unconstrained numerical optimization. Although not usually interpreted thus, these are learning algorithms that fit a local quadratic approximation to the objective function. We show that many, including the most popular, quasi-Newton methods can be interpreted as approximations of Bayesian linear regression under varying prior assumptions. This new notion elucidates some shortcomings of classical algorithms, and lights the way to a novel nonparametric quasi-Newton method, which is able to make more efficient use of available information at computational cost similar to its predecessors.},
volume = {14},
number = {1},
pages = {843--865},
month = mar,
year = {2013},
slug = {hennig13},
author = {Hennig, Philipp and Kiefel, Martin},
url = {http://www.jmlr.org/papers/volume14/hennig13a/hennig13a.pdf},
month_numeric = {3}
}