Back
Quasi-Newton Methods: A New Direction
Four decades after their invention, quasi- Newton methods are still state of the art in unconstrained numerical optimization. Although not usually interpreted thus, these are learning algorithms that fit a local quadratic approximation to the objective function. We show that many, including the most popular, quasi-Newton methods can be interpreted as approximations of Bayesian linear regression under varying prior assumptions. This new notion elucidates some shortcomings of classical algorithms, and lights the way to a novel nonparametric quasi-Newton method, which is able to make more efficient use of available information at computational cost similar to its predecessors.
@inproceedings{optimization, title = {Quasi-Newton Methods: A New Direction}, booktitle = {Proceedings of the 29th International Conference on Machine Learning}, abstract = {Four decades after their invention, quasi- Newton methods are still state of the art in unconstrained numerical optimization. Although not usually interpreted thus, these are learning algorithms that fit a local quadratic approximation to the objective function. We show that many, including the most popular, quasi-Newton methods can be interpreted as approximations of Bayesian linear regression under varying prior assumptions. This new notion elucidates some shortcomings of classical algorithms, and lights the way to a novel nonparametric quasi-Newton method, which is able to make more efficient use of available information at computational cost similar to its predecessors.}, pages = {25--32}, series = {ICML '12}, editors = {John Langford and Joelle Pineau}, publisher = {Omnipress}, address = {New York, NY, USA}, month = jul, year = {2012}, slug = {optimization}, author = {Hennig, Philipp and Kiefel, Martin}, url = {http://www.is.tuebingen.mpg.de/fileadmin/user_upload/files/publications/2012/Hennig_Kiefel_ICML2012.pdf}, month_numeric = {7} }