Document Zbl 1397.65085

<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <title>Document Zbl 1397.65085 - zbMATH Open</title> <meta name="viewport" content="width=device-width, minimum-scale=0.1, maximum-scale=5.0"> <meta name="robots" content="noarchive, noindex"> <meta name="referrer" content="origin-when-cross-origin"> <link href="https://static.zbmath.org/contrib/bootstrap/v3.3.7/css/bootstrap.min.css" rel="stylesheet" media="screen,print"> <link href="https://static.zbmath.org/contrib/bootstrap/v3.3.7/css/bootstrap-theme.min.css" rel="stylesheet" media="screen,print"> <link href="https://static.zbmath.org/contrib/bootstrap-lightbox/v0.7.0/bootstrap-lightbox.min.css" rel="stylesheet" media="screen,print"> <link rel="stylesheet" href="https://static.zbmath.org/contrib/bootstrap-select/v1.13.14/css/bootstrap-select.min.css"> <link href="/static/css/smoothness/jquery-ui-1.10.1.custom.min.css" rel="stylesheet" media="screen"> <link href="/static/styles.css?v=20241024" rel="stylesheet" media="screen,print"> <link href="https://static.zbmath.org/zbMathJax/v0.1.38/zbmathjax.css" rel="stylesheet" media="screen,print"> <link rel="shortcut icon" href="/static/zbmath.ico"> <script type="application/ld+json"> { "@context": "http://schema.org", "@type": "Organization", "url": "https://zbmath.org/", "logo": "https://zbmath.org/static/zbMATH.png" } </script> </head> <body> <div id="line"></div> <span id="clear" style="cursor: pointer;">×</span> <div id="page"> <div id="head"> <nav id="menu" class="navbar navbar-default"> <div class="container-fluid"> <div class="navbar-header"> <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#zbnav" aria-expanded="false"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> <a class="navbar-brand" href="#"> <img class="logo" src="/static/zbmath.gif" alt="zbMATH Open logo"> </a> </div> <div id="zbnav" class="collapse navbar-collapse"> <ul class="nav navbar-nav pages"> <li class="about"> <a href="/about/">About</a> </li> <li class="frequently-asked-questions"> <a href="/frequently-asked-questions/">FAQ</a> </li> <li class="general-help"> <a href="/general-help/">General Help</a> </li> <li class="reviewer-service"> <a href="https://zbmath.org/reviewer-service/" target="_self" >Reviewer Service</a> </li> <li> <a href="/tools-and-resources/">Tools & Resources</a> </li> <li class="contact"> <a href="/contact/">Contact</a> </li> </ul> <ul class="nav navbar-nav navbar-right prefs"> <li class="preferences dropdown"> <a data-toggle="dropdown" href="#">Preferences <i class="caret"></i></a> <ul class="dropdown-menu preferences"> <li> <form id="preferences" class="navbar-form" method="post" action="/preferences/" onsubmit="return confirm('This website uses cookies for the purposes of storing preference information on your device. Do you agree to this?\n\nPlease refer to our Privacy Policy to learn more about our use of cookies.')" > <input type="hidden" name="path" value="/1397.65085?"> <span class=""> <label class="title">Search Form</label> <div class="form-group"> <input id="search-multi-line" type="radio" name="search" value="multi-line" checked> <label for="search-multi-line" class="radio">Multi-Line Search (default)</label> </div> <div class="form-group"> <input id="search-one-line" type="radio" name="search" value="one-line"> <label for="search-one-line" class="radio">One-Line Search</label> </div> </span> <span class="count"> <label class="title">Hits per Page</label> <div class="form-group"> <input id="count-10" type="radio" name="count" value="10"> <label for="count-10" class="radio">10</label> </div> <div class="form-group"> <input id="count-20" type="radio" name="count" value="20"> <label for="count-20" class="radio">20</label> </div> <div class="form-group"> <input id="count-50" type="radio" name="count" value="50"> <label for="count-50" class="radio">50</label> </div> <div class="form-group"> <input id="count-100" type="radio" name="count" value="100" checked> <label for="count-100" class="radio">100 (default)</label> </div> <div class="form-group"> <input id="count-200" type="radio" name="count" value="200"> <label for="count-200" class="radio">200</label> </div> </span> <span class="format"> <label class="title">Display Format</label> <div class="form-group"> <input id="format-mathjax" type="radio" name="format" value="mathjax" checked> <label for="format-mathjax" class="radio">MathJax (default)</label> </div> <div class="form-group"> <input id="format-amstex" type="radio" name="format" value="latex"> <label for="format-amstex" class="radio">LaTeX</label> </div> </span> <span class="ranking"> <label class="title">Documents Sorting</label> <div class="form-group"> <input id="documents-ranking-default" type="radio" name="documents_ranking" value="date" checked> <label for="documents-ranking-default" class="radio">Newest first (default)</label> </div> <div class="form-group"> <input id="documents-ranking-references" type="radio" name="documents_ranking" value="references"> <label for="documents-ranking-references" class="radio">Citations</label> </div> <div class="form-group"> <input id="documents-ranking-relevance" type="radio" name="documents_ranking" value="relevance"> <label for="documents-ranking-relevance" class="radio">Relevance</label> </div> </span> <span class="ranking"> <label class="title">Authors Sorting</label> <div class="form-group"> <input id="authors-ranking-default" type="radio" name="authors_ranking" value="alpha" checked> <label for="authors-ranking-default" class="radio">Alphabetically (default)</label> </div> <div class="form-group"> <input id="authors-ranking-references" type="radio" name="authors_ranking" value="references"> <label for="authors-ranking-references" class="radio">Citations</label> </div> </span> <span class="ranking"> <label class="title">Serials Sorting</label> <div class="form-group"> <input id="serials-ranking-default" type="radio" name="serials_ranking" value="alpha" checked> <label for="serials-ranking-default" class="radio">Alphabetically (default)</label> </div> <div class="form-group"> <input id="serials-ranking-references" type="radio" name="serials_ranking" value="references"> <label for="serials-ranking-references" class="radio">Citations</label> </div> </span> <span class="ranking"> <label class="title">Software Sorting</label> <div class="form-group"> <input id="software-ranking-default" type="radio" name="software_ranking" value="references" checked> <label for="software-ranking-default" class="radio">Citations (default)</label> </div> <div class="form-group"> <input id="software-ranking-alpha" type="radio" name="software_ranking" value="alpha"> <label for="software-ranking-alpha" class="radio">Alphabetically</label> </div> </span> <button type="submit" class="btn btn-default">OK</button> <div class="clearfix"> </form> </li> </ul> </li> </ul> </div> </div> </nav> <div id="tabs"> <h1 class="logo"> <a class="logo" href="/"> <img class="logo" src="/static/zbmath.gif" alt="zbMATH Open — the first resource for mathematics" > </a> </h1> <nav> <ul class="nav nav-tabs"> <li class="tab-documents active"> <a href="/">Documents</a> </li> <li class="tab-authors"> <a href="/authors/">Authors</a> </li> <li class="tab-serials"> <a href="/serials/">Serials</a> </li> <li class="tab-software"> <a href="/software/">Software</a> </li> <li class="tab-classification"> <a href="/classification/">Classification</a> </li> <li class="tab-formulae"> <a href="/formulae/">Formulæ</a> </li> </ul> </nav> <div class="clearfix"></div> </div> <div class="content-fixed"> <div class="content-formular"> <div style="display: none;"> <div class="row ml-0"id="multi-line-new-line" style="display: none;"> <div class="col-xs-12 form-inline multi-line"> <select class="form-control multi-line-field multi-line-selectpicker" name="ml-0-f" aria-label="field"> <option data-type="input" value="any" selected>Anywhere</option> <option data-type="input" value="au">Authors</option> <option data-type="input" value="ti">Title</option> <option data-type="input" value="py">Year</option> <option data-type="range" value="pyr">Year Range</option> <option data-type="input" value="cc">MSC</option> <option data-type="input" value="cc1">MSC Primary</option> <option data-type="input" value="so">Source / Journal</option> <option data-type="input" value="pu">Publisher</option> <option data-type="input" value="la">Language</option> <option data-type="input" value="ab">Summary / Review</option> <option data-type="input" value="rv">Reviewer</option> <option data-type="input" value="an">zbMATH ID</option> <option data-type="input" value="en">External ID</option> <option data-type="input" value="ut">Keywords</option> <option data-type="input" value="sw">Software</option> <option data-type="input" value="br">Biographic Ref</option> <option data-type="input" value="rft">Reference Text</option> <option data-type="multiselect-db" value="db">Database</option> <option data-divider="true"></option> <option data-function="remove-line" data-content='<span class="glyphicon glyphicon-minus" aria-hidden="true"></span> remove line' value="any">remove line</option> </select><input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value"><select class="form-control multi-line-operators multi-line-selectpicker" name="ml-0-op" aria-label="operator"> <option value="and" selected>AND</option> <option value="andnot">AND NOT</option> <option value="or">OR</option> </select></div> </div> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input"> <span class="multi-line-value" id="multi-line-type-range"><span style="padding-left: 5px;">from</span> <input name="ml-0-v1" class="form-control multi-line-input" type="text" value="" aria-label="value"> until <input name="ml-0-v2" class="form-control multi-line-input" type="text" value="" aria-label="value"></span> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input-la" placeholder="use name or ISO code"> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input-rv" placeholder="enter name or zbMATH reviewer number"> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input-an" placeholder="Zbl, JFM or ERAM number"> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input-en" placeholder="e.g. DOI, ISBN, arXiv ID"> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input-sw" placeholder="use * to find all documents using software"> <input name="ml-0-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value" id="multi-line-type-input-br" placeholder="find documents about the life or work of a person"> <span class="multi-line-value" id="multi-line-type-multiselect-db"> <select class="multi-line-selectpicker" data-width="100%" multiple> <option value="zbl">Zbl</option> <option value="arxiv">arXiv</option> <option value="jfm">JFM</option> <option value="eram">ERAM</option> </select> <input type="hidden" class="multi-line-input" name="ml-0-v" value=""> </span> </div> <form name="documents" method="GET" action="/" autocomplete="off"> <div class="documents multi-line" style="display: none;"> <div class="forms"> <ul class="nav forms"> <li class="one-line"> <span tabindex="0" class="glyphicon glyphicon-question-sign" title="One-Line Search allows for free logical combinations of search fields" aria-label="One-Line Search allows for free logical combinations of search fields" data-placement="bottom"></span> <a style="display: inline-block;" href="#">One-Line Search <span class="glyphicon glyphicon-search"></span></a> </li> </ul> </div> <div class="clearfix"></div> <div class="container-fluid"> <input type="hidden" id="multi-line-ml" name="ml" value="3"> <div id="multi-line-row-wrapper"> <div class="row ml-1"> <div class="col-xs-12 form-inline multi-line"> <select class="form-control multi-line-field multi-line-selectpicker" name="ml-1-f" aria-label="field"> <option data-type="input" value="any" selected>Anywhere</option> <option data-type="input" value="au">Authors</option> <option data-type="input" value="ti">Title</option> <option data-type="input" value="py">Year</option> <option data-type="range" value="pyr">Year Range</option> <option data-type="input" value="cc">MSC</option> <option data-type="input" value="cc1">MSC Primary</option> <option data-type="input" value="so">Source / Journal</option> <option data-type="input" value="pu">Publisher</option> <option data-type="input" value="la">Language</option> <option data-type="input" value="ab">Summary / Review</option> <option data-type="input" value="rv">Reviewer</option> <option data-type="input" value="an">zbMATH ID</option> <option data-type="input" value="en">External ID</option> <option data-type="input" value="ut">Keywords</option> <option data-type="input" value="sw">Software</option> <option data-type="input" value="br">Biographic Ref</option> <option data-type="input" value="rft">Reference Text</option> <option data-type="multiselect-db" value="db">Database</option> <option data-divider="true"></option> <option data-function="remove-line" data-content='<span class="glyphicon glyphicon-minus" aria-hidden="true"></span> remove line' value="any">remove line</option> </select><input name="ml-1-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value"><select class="form-control multi-line-operators multi-line-selectpicker" name="ml-1-op" aria-label="operator"> <option value="and" selected>AND</option> <option value="andnot">AND NOT</option> <option value="or">OR</option> </select></div> </div> <div class="row ml-2"> <div class="col-xs-12 form-inline multi-line"> <select class="form-control multi-line-field multi-line-selectpicker" name="ml-2-f" aria-label="field"> <option data-type="input" value="any">Anywhere</option> <option data-type="input" value="au" selected>Authors</option> <option data-type="input" value="ti">Title</option> <option data-type="input" value="py">Year</option> <option data-type="range" value="pyr">Year Range</option> <option data-type="input" value="cc">MSC</option> <option data-type="input" value="cc1">MSC Primary</option> <option data-type="input" value="so">Source / Journal</option> <option data-type="input" value="pu">Publisher</option> <option data-type="input" value="la">Language</option> <option data-type="input" value="ab">Summary / Review</option> <option data-type="input" value="rv">Reviewer</option> <option data-type="input" value="an">zbMATH ID</option> <option data-type="input" value="en">External ID</option> <option data-type="input" value="ut">Keywords</option> <option data-type="input" value="sw">Software</option> <option data-type="input" value="br">Biographic Ref</option> <option data-type="input" value="rft">Reference Text</option> <option data-type="multiselect-db" value="db">Database</option> <option data-divider="true"></option> <option data-function="remove-line" data-content='<span class="glyphicon glyphicon-minus" aria-hidden="true"></span> remove line' value="any">remove line</option> </select><input name="ml-2-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value"><select class="form-control multi-line-operators multi-line-selectpicker" name="ml-2-op" aria-label="operator"> <option value="and" selected>AND</option> <option value="andnot">AND NOT</option> <option value="or">OR</option> </select></div> </div> <div class="row ml-3"> <div class="col-xs-12 form-inline multi-line"> <select class="form-control multi-line-field multi-line-selectpicker" name="ml-3-f" aria-label="field"> <option data-type="input" value="any">Anywhere</option> <option data-type="input" value="au">Authors</option> <option data-type="input" value="ti" selected>Title</option> <option data-type="input" value="py">Year</option> <option data-type="range" value="pyr">Year Range</option> <option data-type="input" value="cc">MSC</option> <option data-type="input" value="cc1">MSC Primary</option> <option data-type="input" value="so">Source / Journal</option> <option data-type="input" value="pu">Publisher</option> <option data-type="input" value="la">Language</option> <option data-type="input" value="ab">Summary / Review</option> <option data-type="input" value="rv">Reviewer</option> <option data-type="input" value="an">zbMATH ID</option> <option data-type="input" value="en">External ID</option> <option data-type="input" value="ut">Keywords</option> <option data-type="input" value="sw">Software</option> <option data-type="input" value="br">Biographic Ref</option> <option data-type="input" value="rft">Reference Text</option> <option data-type="multiselect-db" value="db">Database</option> <option data-divider="true"></option> <option data-function="remove-line" data-content='<span class="glyphicon glyphicon-minus" aria-hidden="true"></span> remove line' value="any">remove line</option> </select><input name="ml-3-v" class="form-control multi-line-value multi-line-input" type="text" value="" aria-label="value"><div id="multi-line-plus"> <a href="#"><span class="glyphicon glyphicon-plus" aria-hidden="true"></span> add line</a> </div></div> </div> </div> <div class="row"> <div class="col-xs-12 form-inline"> <div class="form-group field checkboxes-wrapper" id="checkboxes-wrapper-test" style="visibility: hidden; position: fixed;"> <label>Document Type:</label> <div class="checkboxes"> <div class="slider"> <label title="search for Articles in Journals"> <input type="checkbox" class="form-control" value="j" checked> <span tabindex="0"><small></small></span> Journal Articles </label> </div> <div class="slider"> <label title="search for Articles in Conference Proceedings and Collected Volumes"> <input type="checkbox" class="form-control" value="a" checked> <span tabindex="0"><small></small></span> Collection Articles </label> </div> <div class="slider"> <label title="search for Monographs, Proceedings, Dissertations etc."> <input type="checkbox" class="form-control" value="b" checked> <span tabindex="0"><small></small></span> Books </label> </div> <div class="slider"> <label title="search for arXiv Preprints"> <input type="checkbox" class="form-control" value="p" checked> <span tabindex="0"><small></small></span> arXiv Preprints </label> </div> </div> </div> <div class="form-group field checkboxes-wrapper" id="checkboxes-wrapper-real"> <label>Document Type:</label> <div class="checkboxes"> <div class="slider"> <label for="dt-j" title="search for Articles in Journals"> <input type="checkbox" id="dt-j" name="dt" class="form-control" value="j" checked> <span tabindex="0"><small></small></span> Journal Articles </label> </div> <div class="slider"> <label for="dt-a" title="search for Articles in Conference Proceedings and Collected Volumes"> <input type="checkbox" id="dt-a" name="dt" class="form-control" value="a" checked> <span tabindex="0"><small></small></span> Collection Articles </label> </div> <div class="slider"> <label for="dt-b" title="search for Monographs, Proceedings, Dissertations etc."> <input type="checkbox" id="dt-b" name="dt" class="form-control" value="b" checked> <span tabindex="0"><small></small></span> Books </label> </div> <div class="slider"> <label for="dt-p" title="search for arXiv Preprints"> <input type="checkbox" id="dt-p" name="dt" class="form-control" value="p" checked> <span tabindex="0"><small></small></span> arXiv Preprints </label> </div> </div> </div> </div> </div> <div class="row"> <div class="col-xs-12 buttons"> <a tabindex="0" class="btn btn-default clear-all">Reset all <span class="glyphicon glyphicon-remove"></span></a> <div class="submit"> <button class="btn btn-default search" type="submit">Search <span class="glyphicon glyphicon-search"></span></button> </div> </div> </div> </div> </div> </form> <form class="form-inline" name="documents" method="GET" action="/"> <div class="documents one-line" style="display: block;"> <div class="forms"> <ul class="nav forms"> <li class="multi-line"><a href="#">New Multi-Line Search <span class="glyphicon glyphicon-list"></span></a></li> </ul> </div> <div id="search-row" class="input-group box"> <span> <div id="search-field"> <input class="query form-control" type="text" name="q" value="an:1397.65085" aria-label="Search for documents" placeholder="Search for documents" autocomplete="off"> </div> <div class="search-buttons input-group-btn"> <div class="btn-group"> <button class="btn btn-default search" type="submit"><span class="virtual">Search</span> <span class="glyphicon glyphicon-search" style="top: 2px;"></span></button> </div> </div> </span> <span> <div class="search-buttons input-group-btn"> <div class="btn-group"> <div class="btn-group fields"> <button class="btn btn-default dropdown-toggle" data-toggle="dropdown">Fields <i class="caret"></i></button> <ul id="fields" class="dropdown-menu pull-right"> <li><a href="#"><span class="token item">any:</span><span> </span>anywhere (default)</a></li> <li><a href="#"><span class="token item">ab:</span><span>  </span>review text</a></li> <li><a href="#"><span class="token item">an:</span><span>  </span>zbmath id</a></li> <li><a href="#"><span class="token item">any:</span><span>  </span>anywhere</a></li> <li><a href="#"><span class="token item">au:</span><span>  </span>contributor name</a></li> <li><a href="#"><span class="token item">br:</span><span>  </span>biographic reference name</a></li> <li><a href="#"><span class="token item">cc:</span><span>  </span>msc title</a></li> <li><a href="#"><span class="token item">dt:</span><span>  </span>document type</a></li> <li><a href="#"><span class="token item">doi:</span><span>  </span>doi</a></li> <li><a href="#"><span class="token item">en:</span><span>  </span>external id</a></li> <li><a href="#"><span class="token item">la:</span><span>  </span>language</a></li> <li><a href="#"><span class="token item">pu:</span><span>  </span>publisher</a></li> <li><a href="#"><span class="token item">py:</span><span>  </span>year</a></li> <li><a href="#"><span class="token item">rv:</span><span>  </span>reviewer name</a></li> <li><a href="#"><span class="token item">so:</span><span>  </span>source</a></li> <li><a href="#"><span class="token item">sw:</span><span>  </span>software name</a></li> <li><a href="#"><span class="token item">ti:</span><span>  </span>title</a></li> <li><a href="#"><span class="token item">ut:</span><span>  </span>keyword</a></li> </ul> </div> <div class="btn-group operators"> <button class="btn btn-default dropdown-toggle" data-toggle="dropdown">Operators <i class="caret"></i></button> <ul id="operators" class="dropdown-menu pull-right"> <li><a href="#"><span class="token">a <span class="item">&</span> b </span><span> </span>logical and (default)</a></li> <li><a href="#"><span class="token">a <span class="item">|</span> b </span><span> </span>logical or</a></li> <li><a href="#"><span class="token"><span class="item">!</span>ab   </span><span> </span>logical not</a></li> <li><a href="#"><span class="token">abc<span class="item">*</span>  </span><span> </span>right wildcard</a></li> <li><a href="#"><span class="token"><span class="item">"</span>ab c<span class="item">"</span></span><span> </span>phrase</a></li> <li><a href="#"><span class="token"><span class="item">(</span>ab c<span class="item">)</span></span><span> </span>parentheses</a></li> </ul> </div> </div> </div> <div class="special"> <ul class="nav help-button"> <li class="dropdown pull-right"> <a href="#">Help <i class="caret"></i></a> </li> </ul> </div> </span> </div> <div class="help"><h2>Examples</h2> <div id="help-terms" role="table"> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=Geometry">Geometry</a></span> <span class="search-explanation" role="cell" role="cell">Search for the term <em>Geometry</em> in <strong>any</strong> field. Queries are <strong>case-independent</strong>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=Funct%2A">Funct*</a></span> <span class="search-explanation" role="cell"><strong>Wildcard</strong> queries are specified by <strong><u>*</u></strong> (e.g. <em>functions</em>, <em>functorial</em>, etc.). Otherwise the search is <strong>exact</strong>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=%22Topological+group%22">"Topological group"</a></span> <span class="search-explanation" role="cell"><strong>Phrases</strong> (multi-words) should be set in <u>"</u>straight quotation marks<u>"</u>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=au%3A+Bourbaki+%26+ti%3A+Algebra">au: Bourbaki & ti: Algebra</a></span> <span class="search-explanation" role="cell">Search for <strong><u>au</u>thor</strong> and <strong><u>ti</u>tle</strong>. The <strong>and-operator &</strong> is default and can be omitted.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=Chebyshev+%7C+Tschebyscheff">Chebyshev | Tschebyscheff</a></span> <span class="search-explanation" role="cell">The <strong>or-operator |</strong> allows to search for <em>Chebyshev</em> or <em>Tschebyscheff</em>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=Quasi%2A+map%2A+py%3A+1989">Quasi* map* py: 1989</a></span> <span class="search-explanation" role="cell">The resulting documents have <strong><u>p</u>ublication <u>y</u>ear</strong> <em>1989</em>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=so%3A+Eur%2A+J%2A+Mat%2A+Soc%2A+cc%3A+14">so: Eur* J* Mat* Soc* cc: 14</a></span> <span class="search-explanation" role="cell">Search for publications in a particular <strong><u>so</u>urce</strong> with a <strong>Mathematics Subject <u>C</u>lassification <u>c</u>ode (<u>cc</u>)</strong> in <em>14</em>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=%22Partial+diff%2A+eq%2A%22+%21+elliptic">"Partial diff* eq*" ! elliptic</a></span> <span class="search-explanation" role="cell">The <strong>not</strong>-operator <strong>!</strong> eliminates all results containing the word <em>elliptic</em>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=dt%3A+b+%26+au%3A+Hilbert">dt: b & au: Hilbert</a></span> <span class="search-explanation" role="cell">The <strong><u>d</u>ocument <u>t</u>ype</strong> is set to books; alternatively: <u>j</u> for <strong>journal articles</strong>, <u>a</u> for <strong>book articles</strong>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=py%3A+2000-2015+cc%3A+%2894A+%7C+11T%29">py: 2000-2015 cc: (94A | 11T)</a></span> <span class="search-explanation" role="cell">Number <strong>ranges</strong> are accepted. Terms can be grouped within <strong><u>(</u>parentheses<u>)</u></strong>.</span> </div> <div class="help-item" role="row"> <span class="search-example" role="rowheader"><a href="/?q=la%3A+chinese">la: chinese</a></span> <span class="search-explanation" role="cell">Find documents in a given <strong><u>la</u>nguage</strong>. <a href="http://en.wikipedia.org/wiki/ISO_639-1">ISO 639-1</a> language codes can also be used.</span> </div> </div> <div id="help-fields"> <h2>Fields</h2> <table> <tr> <td class="nowrap padding" role="rowheader"><strong>any</strong></td> <td class="padding">anywhere</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>an</strong></td> <td class="padding">internal document identifier</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>au</strong></td> <td class="padding">author, editor</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>ai</strong></td> <td class="padding">internal author identifier</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>ti</strong></td> <td class="padding">title</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>la</strong></td> <td class="padding">language</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>so</strong></td> <td class="padding">source</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>ab</strong></td> <td class="padding">review, abstract</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>py</strong></td> <td class="padding">publication year</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>rv</strong></td> <td class="padding">reviewer</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>cc</strong></td> <td class="padding">MSC code</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>ut</strong></td> <td class="padding">uncontrolled term</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>dt</strong></td> <td class="padding" colspan="4">document type (<strong>j</strong>: journal article; <strong>b</strong>: book; <strong>a</strong>: book article)</td> </tr> </table> </div> <div id="help-operators"> <h2>Operators</h2> <table> <tr> <td class="nowrap padding" role="rowheader">a <strong>&</strong> b</td> <td class="padding">logic and</td> </tr> <tr> <td class="nowrap padding" role="rowheader">a <strong>|</strong> b</td> <td class="padding">logic or</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>!</strong>ab</td> <td class="padding">logic not</td> </tr> <tr> <td class="nowrap padding" role="rowheader">abc<strong>*</strong></td> <td class="padding">right wildcard</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>"</strong>ab c<strong>"</strong></td> <td class="padding">phrase</td> </tr> <tr> <td class="nowrap padding" role="rowheader"><strong>(</strong>ab c<strong>)</strong></td> <td class="padding">parentheses</td> </tr> </table> </div> <p> See also our <a href="/general-help/">General Help</a>. </p></div> </div> </form> <div class="clearfix"></div> </div> <div class="content-shadow"></div> </div> </div> <div id="body"> <div id="main"> <div class="messages"> </div> <div id="documents"> <div class="content-main"> <div class="content-item"><div class="item"> <article> <div class="author"><a href="/authors/bottou.leon" title="Author Profile">Bottou, Léon</a>; <a href="/authors/curtis.frank-e" title="Author Profile">Curtis, Frank E.</a>; <a href="/authors/nocedal.jorge" title="Author Profile">Nocedal, Jorge</a></div> <h2 class="title"> <strong>Optimization methods for large-scale machine learning.</strong> <i>(English)</i> <a class="label nowrap" href="/1397.65085">Zbl 1397.65085</a> </h2> <div class="source"> <a href="/serials/1639" title="Journal Profile">SIAM Rev.</a> <a href="/?q=in%3A399343" title="Articles in this Issue">60, No. 2, 223-311 (2018)</a>. </div> <div class="abstract">This paper provides a review and commentary on the past, present, and future of numerical optimization algorithms in the context of machine learning applications. Through case studies on text classification and the training of deep neural networks, is discussed how optimization problems arise in machine learning and what makes them challenging. A main theme of this study is that large-scale machine learning represents a distinctive setting in which the stochastic gradient method has traditionally played a central role while conventional gradient-based nonlinear optimization techniques typically falter. Based on this viewpoint, a comprehensive theory of a straightforward, yet versatile stochastic gradient algorithm, discussed its practical behavior, and highlight opportunities for designing algorithms with improved performance is presented. This leads to a discussion about the next generation of optimization methods for large-scale machine learning, including an investigation of two main streams of research on techniques that diminish noise in the stochastic directions and methods that make use of second-order derivative approximations.<div class="reviewer"> Reviewer: <a href="/authors/?q=rv%3A4143">Tiit Riismaa (Tallinn)</a></div> <div class="clearfix"></div></div> <div class="clear"></div> <br> <div class="citations"><div class="clear"><a href="/?q=ci%3A6870204">Cited in <strong>1</strong> Review</a></div><div class="clear"><a href="/?q=rf%3A6870204">Cited in <strong>440</strong> Documents</a></div></div> <div class="classification"> <h3>MSC:</h3> <table><tr> <td> <a class="mono" href="/classification/?q=cc%3A65K05" title="MSC2020">65K05</a> </td> <td class="space"> Numerical mathematical programming methods </td> </tr><tr> <td> <a class="mono" href="/classification/?q=cc%3A68Q25" title="MSC2020">68Q25</a> </td> <td class="space"> Analysis of algorithms and problem complexity </td> </tr><tr> <td> <a class="mono" href="/classification/?q=cc%3A68T05" title="MSC2020">68T05</a> </td> <td class="space"> Learning and adaptive systems in artificial intelligence </td> </tr><tr> <td> <a class="mono" href="/classification/?q=cc%3A90C06" title="MSC2020">90C06</a> </td> <td class="space"> Large-scale problems in mathematical programming </td> </tr><tr> <td> <a class="mono" href="/classification/?q=cc%3A90C30" title="MSC2020">90C30</a> </td> <td class="space"> Nonlinear programming </td> </tr><tr> <td> <a class="mono" href="/classification/?q=cc%3A49Q10" title="MSC2020">49Q10</a> </td> <td class="space"> Optimization of shapes other than minimal surfaces </td> </tr></table> </div><div class="keywords"> <h3>Keywords:</h3><a href="/?q=ut%3Anumerical+optimization">numerical optimization</a>; <a href="/?q=ut%3Amachine+learning">machine learning</a>; <a href="/?q=ut%3Astochastic+gradient+methods">stochastic gradient methods</a>; <a href="/?q=ut%3Aalgorithm+complexity+analysis">algorithm complexity analysis</a>; <a href="/?q=ut%3Anoise+reduction+methods">noise reduction methods</a>; <a href="/?q=ut%3Asecond-order+methods">second-order methods</a></div> <div class="software"> <h3>Software:</h3><a href="/software/22202">AdaGrad</a>; <a href="/software/38522">AlexNet</a>; <a href="/software/4880">LIBLINEAR</a>; <a href="/software/22204">RMSprop</a>; <a href="/software/4714">TRON</a>; <a href="/software/8752">Pegasos</a>; <a href="/software/7279">RCV1</a>; <a href="/software/12839">TFOCS</a>; <a href="/software/11795">QUIC</a>; <a href="/software/39677">Saga</a>; <a href="/software/28396">HOGWILD</a>; <a href="/software/19411">SGD-QN</a>; <a href="/software/22205">Adam</a>; <a href="/software/39429">ADADELTA</a>; <a href="/software/21105">ImageNet</a>; <a href="/software/3229">L-BFGS</a></div>  <div class="modal fade" id="metadataModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> <h4 class="modal-title" id="myModalLabel">Cite</h4> </div> <div class="modal-body"> <div class="form-group"> <label for="select-output" class="control-label">Format</label> <select id="select-output" class="form-control" aria-label="Select Metadata format"></select> </div> <div class="form-group"> <label for="metadataText" class="control-label">Result</label> <textarea class="form-control" id="metadataText" rows="10" style="min-width: 100%;max-width: 100%"></textarea> </div> <div id="metadata-alert" class="alert alert-danger" role="alert" style="display: none;">  </div> </div> <div class="modal-footer"> <button type="button" class="btn btn-primary" onclick="copyMetadata()">Copy to clipboard</button> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> </div> </div> </div> <div class="functions clearfix"> <div class="function">  <a type="button" class="btn btn-default btn-xs pdf" data-toggle="modal" data-target="#metadataModal" data-itemtype="Zbl" data-itemname="Zbl 1397.65085" data-ciurl="/ci/06870204" data-biburl="/bibtex/06870204.bib" data-amsurl="/amsrefs/06870204.bib" data-xmlurl="/xml/06870204.xml" > Cite </a> <a class="btn btn-default btn-xs pdf" data-container="body" type="button" href="/pdf/06870204.pdf" title="Zbl 1397.65085 as PDF">Review PDF</a> </div> <div class="fulltexts"> <span class="fulltext">Full Text:</span> <a class="btn btn-default btn-xs" type="button" href="https://doi.org/10.1137/16M1080173" aria-label="DOI for “Optimization methods for large-scale machine learning”" title="10.1137/16M1080173">DOI</a> <a class="btn btn-default btn-xs" type="button" href="https://arxiv.org/abs/1606.04838"title="Note: arXiv document may differ from published version">arXiv</a> <a class="btn btn-default btn-xs" type="button" href="https://www.osti.gov/biblio/1541717" title="Full Text Link">Link</a> </div> <div class="sfx" style="float: right;"> </div> </div> <div class="references"> <h3>References:</h3> <table><tr> <td>[1]</td> <td class="space">A. Agarwal, P. L. Bartlett, P. Ravikumar, and M. J. Wainwright, {\it Information-theoretic lower bounds on the oracle complexity of stochastic convex optimization}, IEEE Trans. Inform. Theory, 58 (2012), pp. 3235-3249. · <a href="/1365.94132" class="nowrap">Zbl 1365.94132</a></td> </tr><tr> <td>[2]</td> <td class="space">N. Agarwal, B. Bullins, and E. Hazan, {\it Second-order stochastic optimization for machine learning in linear time}, J. Mach. Learn. Res., 18 (2017), pp. 4148-4187. · <a href="/1441.90115" class="nowrap">Zbl 1441.90115</a></td> </tr><tr> <td>[3]</td> <td class="space">Z. Allen Zhu and E. Hazan, {\it Optimal black-box reductions between optimization objectives}, in Advances in Neural Information Processing Systems 29, D. D. Lee, M. Sugiyama, U. V. Luxburg, I. Guyon, and R. Garnett, eds., 2016, pp. 1606-1614.</td> </tr><tr> <td>[4]</td> <td class="space">S.-I. Amari, {\it A theory of adaptive pattern classifiers}, IEEE Trans. Electronic Computers, EC-16 (1967), pp. 299-307. · <a href="/0189.50101" class="nowrap">Zbl 0189.50101</a></td> </tr><tr> <td>[5]</td> <td class="space">S.-I. Amari, {\it Natural gradient works efficiently in learning}, Neural Comput., 10 (1998), pp. 251-276.</td> </tr><tr> <td>[6]</td> <td class="space">S.-I. Amari and H. Nagaoka, {\it Methods of Information Geometry}, AMS, Providence, RI, 1997.</td> </tr><tr> <td>[7]</td> <td class="space">G. Andrew and J. Gao, {\it Scalable training of $L^1$-regularized log-linear models}, in Proceedings of the 24th International Conference on Machine Learning, ACM, 2007, pp. 33-40.</td> </tr><tr> <td>[8]</td> <td class="space">Y. F. Atchade, G. Fort, and E. Moulines, {\it On Stochastic Proximal Gradient Algorithms}, preprint, , 2014. · <a href="/1433.90199" class="nowrap">Zbl 1433.90199</a></td> </tr><tr> <td>[9]</td> <td class="space">F. Bach, R. Jenatton, J. Mairal, and G. Obozinski, {\it Optimization with sparsity-inducing penalties}, Found. Trends Mach. Learning, 4 (2012), pp. 1-106. · <a href="/06064248" class="nowrap">Zbl 06064248</a></td> </tr><tr> <td>[10]</td> <td class="space">A. Beck and M. Teboulle, {\it A fast iterative shrinkage-thresholding algorithm for linear inverse problems}, SIAM J. Imaging Sci., 2 (2009), pp. 183-202, . · <a href="/1175.94009" class="nowrap">Zbl 1175.94009</a></td> </tr><tr> <td>[11]</td> <td class="space">S. Becker, E. J. Candès, and M. Grant, {\it Templates for convex cone problems with applications to sparse signal recovery}, Math. Program. Comput., 3 (2011), pp. 165-218. · <a href="/1257.90042" class="nowrap">Zbl 1257.90042</a></td> </tr><tr> <td>[12]</td> <td class="space">S. Becker and Y. LeCun, {\it Improving the convergence of back-propagation learning with second-order methods}, in Proceedings of the 1988 Connectionist Models Summer School, D. Touretzky, G. Hinton, and T. Sejnowski, eds., Morgan Kaufman, San Mateo, CA, 1989, pp. 29-37.</td> </tr><tr> <td>[13]</td> <td class="space">D. P. Bertsekas, {\it Nonlinear Programming}, Athena Scientific, Belmont, Massachusetts, 1995. · <a href="/0935.90037" class="nowrap">Zbl 0935.90037</a></td> </tr><tr> <td>[14]</td> <td class="space">D. P. Bertsekas, {\it Incremental least squares methods and the extended Kalman filter}, SIAM J. Optim., 6 (1996), pp. 807-822, . · <a href="/0945.93026" class="nowrap">Zbl 0945.93026</a></td> </tr><tr> <td>[15]</td> <td class="space">D. P. Bertsekas, {\it Convex Optimization Algorithms}, Athena Scientific, Nashua, NH, 2015. · <a href="/1347.90001" class="nowrap">Zbl 1347.90001</a></td> </tr><tr> <td>[16]</td> <td class="space">D. P. Bertsekas and J. N. Tsitsiklis, {\it Parallel and Distributed Computation: Numerical Methods}, Prentice-Hall, Englewood Cliffs, NJ, 1989. · <a href="/0743.65107" class="nowrap">Zbl 0743.65107</a></td> </tr><tr> <td>[17]</td> <td class="space">D. Blatt, A. O. Hero, and H. Gauchman, {\it A convergent incremental gradient method with a constant step size}, SIAM J. Optim., 18 (2007), pp. 29-51, . · <a href="/1154.90015" class="nowrap">Zbl 1154.90015</a></td> </tr><tr> <td>[18]</td> <td class="space">A. Bordes, L. Bottou, and P. Gallinari, {\it SGD-QN: Careful Quasi-Newton stochastic gradient descent}, J. Mach. Learn. Res., 10 (2009), pp. 1737-1754. · <a href="/1235.68130" class="nowrap">Zbl 1235.68130</a></td> </tr><tr> <td>[19]</td> <td class="space">A. Bordes, L. Bottou, P. Gallinari, J. Chang, and S. A. Smith, {\it Erratum: SGDQN is less careful than expected}, J. Mach. Learn. Res., 11 (2010), pp. 2229-2240. · <a href="/1242.68206" class="nowrap">Zbl 1242.68206</a></td> </tr><tr> <td>[20]</td> <td class="space">L. Bottou, {\it Stochastic gradient learning in neural networks}, in Proceedings of Neuro-Nîmes 91, Nimes, France, 1991, EC2.</td> </tr><tr> <td>[21]</td> <td class="space">L. Bottou, {\it Online algorithms and stochastic approximations}, in Online Learning and Neural Networks, D. Saad, ed., Cambridge University Press, Cambridge, UK, 1998. · <a href="/0968.68127" class="nowrap">Zbl 0968.68127</a></td> </tr><tr> <td>[22]</td> <td class="space">L. Bottou, {\it Large-scale machine learning with stochastic gradient descent}, in Proceedings of the 19th International Conference on Computational Statistics (COMPSTAT’2010), Y. Lechevallier and G. Saporta, eds., Paris, 2010, Springer, pp. 177-187. · <a href="/1436.68293" class="nowrap">Zbl 1436.68293</a></td> </tr><tr> <td>[23]</td> <td class="space">L. Bottou and O. Bousquet, {\it The tradeoffs of large scale learning}, in Advances in Neural Information Processing Systems 20, J. C. Platt, D. Koller, Y. Singer, and S. T. Roweis, eds., Curran Associates, Inc., 2008, pp. 161-168.</td> </tr><tr> <td>[24]</td> <td class="space">L. Bottou, F. Fogelman Soulié, P. Blanchet, and J. S. Lienard, {\it Experiments with time delay networks and dynamic time warping for speaker independent isolated digit recognition}, in Proceedings of EuroSpeech 89, Vol. 2, Paris, 1989, pp. 537-540.</td> </tr><tr> <td>[25]</td> <td class="space">L. Bottou and Y. Le Cun, {\it On-line learning for very large datasets}, Appl. Stoch. Models Business Indust., 21 (2005), pp. 137-151. · <a href="/1091.68063" class="nowrap">Zbl 1091.68063</a></td> </tr><tr> <td>[26]</td> <td class="space">O. Bousquet, {\it Concentration Inequalities and Empirical Processes Theory Applied to the Analysis of Learning Algorithms}, Ph.D. thesis, Ecole Polytechnique, 2002.</td> </tr><tr> <td>[27]</td> <td class="space">C. G. Broyden, {\it The convergence of a class of double-rank minimization algorithms}, J. Institute Math. Appl., 6 (1970), pp. 76-90. · <a href="/0223.65023" class="nowrap">Zbl 0223.65023</a></td> </tr><tr> <td>[28]</td> <td class="space">R. H. Byrd, G. M. Chin, W. Neveitt, and J. Nocedal, {\it On the use of stochastic Hessian information in optimization methods for machine learning}, SIAM J. Optim., 21 (2011), pp. 977-995, . · <a href="/1245.65062" class="nowrap">Zbl 1245.65062</a></td> </tr><tr> <td>[29]</td> <td class="space">R. H. Byrd, G. M. Chin, J. Nocedal, and F. Oztoprak, {\it A family of second-order methods for convex $ℓ_1$-regularized optimization}, Math. Program. Ser. A, 159 (2016), pp. 435-467. · <a href="/1350.49046" class="nowrap">Zbl 1350.49046</a></td> </tr><tr> <td>[30]</td> <td class="space">R. H. Byrd, G. M. Chin, J. Nocedal, and Y. Wu, {\it Sample size selection in optimization methods for machine learning}, Math. Program. Ser. B, 134 (2012), pp. 127-155. · <a href="/1252.49044" class="nowrap">Zbl 1252.49044</a></td> </tr><tr> <td>[31]</td> <td class="space">R. H. Byrd, J. Nocedal, and F. Oztoprak, {\it An inexact successive quadratic approximation method for convex L-1 regularized optimization}, Math. Program., 157 (2017), pp. 375-396. · <a href="/1342.49037" class="nowrap">Zbl 1342.49037</a></td> </tr><tr> <td>[32]</td> <td class="space">E. J. Candès and B. Recht, {\it Exact matrix completion via convex optimization}, Found. Comput. Math., 9 (2009), pp. 717-772. · <a href="/1219.90124" class="nowrap">Zbl 1219.90124</a></td> </tr><tr> <td>[33]</td> <td class="space">J.-F. Cardoso, {\it Blind signal separation: Statistical principles}, Proc. IEEE, 9 (1998), pp. 2009-2025.</td> </tr><tr> <td>[34]</td> <td class="space">N. Cesa-Bianchi, A. Conconi, and C. Gentile, {\it On the generalization ability of on-line learning algorithms}, IEEE Trans. Inform. Theory, 50 (2004), pp. 2050-2057. · <a href="/1295.68182" class="nowrap">Zbl 1295.68182</a></td> </tr><tr> <td>[35]</td> <td class="space">A. Choromanska, M. Henaff, M. Mathieu, G. B. Arous, and Y. LeCun, {\it The loss surfaces of multilayer networks}, in Proceedings of the 18th International Conference on Artificial Intelligence and Statistics (AISTATS), 2015, pp. 192-204.</td> </tr><tr> <td>[36]</td> <td class="space">K. L. Chung, {\it On a stochastic approximation method}, Ann. Math. Statist., 25 (1954), pp. 463-483. · <a href="/0059.13203" class="nowrap">Zbl 0059.13203</a></td> </tr><tr> <td>[37]</td> <td class="space">A. R. Conn, N. I. M. Gould, and P. L. Toint, {\it Trust Region Methods}, SIAM, Philadelphia, 2000, . · <a href="/0958.65071" class="nowrap">Zbl 0958.65071</a></td> </tr><tr> <td>[38]</td> <td class="space">C. Cortes and V. N. Vapnik, {\it Support-Vector Networks}, Machine Learning, 20 (1995), pp. 273-297. · <a href="/0831.68098" class="nowrap">Zbl 0831.68098</a></td> </tr><tr> <td>[39]</td> <td class="space">R. Courant and H. Robbins., {\it What Is Mathematics?}, 1st ed., Oxford University Press, 1941. · <a href="/67.0001.05" class="nowrap">JFM 67.0001.05</a></td> </tr><tr> <td>[40]</td> <td class="space">R. Courant and H. Robbins, {\it What Is Mathematics?}, 2nd ed., Oxford University Press, 1996; revised by I. Stewart. · <a href="/0865.00001" class="nowrap">Zbl 0865.00001</a></td> </tr><tr> <td>[41]</td> <td class="space">T. M. Cover, {\it Universal portfolios}, Math. Finance, 1 (1991), pp. 1-29. · <a href="/0900.90052" class="nowrap">Zbl 0900.90052</a></td> </tr><tr> <td>[42]</td> <td class="space">I. Daubechies, M. Defrise, and C. De Mol, {\it An iterative thresholding algorithm for linear inverse problems with a sparsity constraint}, Comm. Pure Appl. Math., 58 (2004), pp. 1413-1457. · <a href="/1077.65055" class="nowrap">Zbl 1077.65055</a></td> </tr><tr> <td>[43]</td> <td class="space">Y. N. Dauphin, H. de Vries, and Y. Bengio, {\it Equilibrated adaptive learning rates for non-convex optimization}, in Advances in Neural Information Processing Systems 28, 2015, pp. 1504-1512.</td> </tr><tr> <td>[44]</td> <td class="space">Y. N. Dauphin, R. Pascanu, Ç. Gülçehre, K. Cho, S. Ganguli, and Y. Bengio, {\it Identifying and attacking the saddle point problem in high-dimensional non-convex optimization}, in Advances in Neural Information Processing Systems 27, 2014, pp. 2933-2941.</td> </tr><tr> <td>[45]</td> <td class="space">J. Dean, G. Corrado, R. Monga, K. Chen, M. Devin, Q. V. Le, M. Z. Mao, M. Ranzato, A. W. Senior, P. A. Tucker, K. Yang, and A. Y. Ng, {\it Large scale distributed deep networks}, in Advances in Neural Information Processing Systems 25, 2012, pp. 1232-1240.</td> </tr><tr> <td>[46]</td> <td class="space">A. Defazio, F. Bach, and S. Lacoste-Julien, {\it SAGA: A fast incremental gradient method with support for non-strongly convex composite objectives}, in Advances in Neural Information Processing Systems 27, 2014, pp. 1646-1654.</td> </tr><tr> <td>[47]</td> <td class="space">R. S. Dembo, S. C. Eisenstat, and T. Steihaug, {\it Inexact Newton methods}, SIAM J. Numer. Anal., 19 (1982), pp. 400-408, . · <a href="/0478.65030" class="nowrap">Zbl 0478.65030</a></td> </tr><tr> <td>[48]</td> <td class="space">A. P. Dempster, N. M. Laird, and D. B. Rubin, {\it Maximum likelihood from incomplete data via the EM algorithm}, J. Roy. Statist. Soc. Ser. B, 39 (1977), pp. 1-38, . · <a href="/0364.62022" class="nowrap">Zbl 0364.62022</a></td> </tr><tr> <td>[49]</td> <td class="space">J. Deng, N. Ding, Y. Jia, A. Frome, K. Murphy, S. Bengio, Y. Li, H. Neven, and H. Adam, {\it Large-scale object classification using label relation graphs}, in Proceedings of the 13th European Conference on Computer Vision (ECCV), Part I, 2014, pp. 48-64.</td> </tr><tr> <td>[50]</td> <td class="space">L. Deng, G. E. Hinton, and B. Kingsbury, {\it New types of deep neural network learning for speech recognition and related applications: An overview}, in IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP 2013), Vancouver, BC, Canada, 2013, pp. 8599-8603.</td> </tr><tr> <td>[51]</td> <td class="space">J. E. Dennis, Jr., and J. J. Moré, {\it A characterization of superlinear convergence and its application to quasi-Newton methods}, Math. Comp., 28 (1974), pp. 549-560. · <a href="/0282.65042" class="nowrap">Zbl 0282.65042</a></td> </tr><tr> <td>[52]</td> <td class="space">J. E. Dennis, Jr., and R. B. Schnabel, {\it Numerical Methods for Unconstrained Optimization and Nonlinear Equations}, Classics Appl. Math. 16, SIAM, Philadelphia, 1996, . · <a href="/0847.65038" class="nowrap">Zbl 0847.65038</a></td> </tr><tr> <td>[53]</td> <td class="space">D. Donoho, {\it De-noising by soft-thresholding}, IEEE Trans. Inform. Theory, 41 (1995), pp. 613-627. · <a href="/0820.62002" class="nowrap">Zbl 0820.62002</a></td> </tr><tr> <td>[54]</td> <td class="space">J. Duchi, E. Hazan, and Y. Singer, {\it Adaptive subgradient methods for online learning and stochastic optimization}, J. Mach. Learn. Res., 12 (2011), pp. 2121-2159. · <a href="/1280.68164" class="nowrap">Zbl 1280.68164</a></td> </tr><tr> <td>[55]</td> <td class="space">R. M. Dudley, {\it Uniform Central Limit Theorems}, Cambridge Stud. Adv. Math. 63, Cambridge University Press, Cambridge, UK, 1999. · <a href="/0951.60033" class="nowrap">Zbl 0951.60033</a></td> </tr><tr> <td>[56]</td> <td class="space">S. T. Dumais, J. C. Platt, D. Hecherman, and M. Sahami, {\it Inductive learning algorithms and representations for text categorization}, in Proceedings of the 1998 ACM CIKM International Conference on Information and Knowledge Management, Bethesda, MD, 1998, pp. 148-155.</td> </tr><tr> <td>[57]</td> <td class="space">J. Eckstein and D. P. Bertsekas, {\it On the Douglas-Rachford splitting method and the proximal point algorithm for maximal monotone operators}, Math. Program., 55 (1992), pp. 293-318. · <a href="/0765.90073" class="nowrap">Zbl 0765.90073</a></td> </tr><tr> <td>[58]</td> <td class="space">R.-E. Fan, K.-W. Chang, C.-J. Hsieh, X.-R. Wang, and C.-J. Lin, {\it Liblinear: A library for large linear classification}, J. Mach. Learn. Res., 9 (2008), pp. 1871-1874. · <a href="/1225.68175" class="nowrap">Zbl 1225.68175</a></td> </tr><tr> <td>[59]</td> <td class="space">M. A. T. Figueiredo, R. D. Nowak, and S. J. Wright, {\it Gradient projection for sparse reconstruction: Application to compressed sensing and other inverse problems}, IEEE J. Selected Topics Signal Process., 1 (2007), pp. 586-597.</td> </tr><tr> <td>[60]</td> <td class="space">R. Fletcher, {\it A new approach to variable metric algorithms}, Comput. J., 13 (1970), pp. 317-322. · <a href="/0207.17402" class="nowrap">Zbl 0207.17402</a></td> </tr><tr> <td>[61]</td> <td class="space">J. E. Freund, {\it Mathematical Statistics}, Prentice-Hall, Englewood Cliffs, NJ, 1962. · <a href="/0142.15001" class="nowrap">Zbl 0142.15001</a></td> </tr><tr> <td>[62]</td> <td class="space">M. P. Friedlander and M. Schmidt, {\it Hybrid deterministic-stochastic methods for data fitting}, SIAM J. Sci. Comput., 34 (2012), pp. A1380-A1405, . · <a href="/1262.90090" class="nowrap">Zbl 1262.90090</a></td> </tr><tr> <td>[63]</td> <td class="space">M. C. Fu, {\it Optimization for simulation: Theory vs. practice}, INFORMS J. Comput., 14 (2002), pp. 192-215. · <a href="/1238.90001" class="nowrap">Zbl 1238.90001</a></td> </tr><tr> <td>[64]</td> <td class="space">D. Gabay and B. Mercier, {\it A dual algorithm for the solution of nonlinear variational problems via finite element approximations}, Comput. Math. Appl., 2 (1976), pp. 17-40. · <a href="/0352.65034" class="nowrap">Zbl 0352.65034</a></td> </tr><tr> <td>[65]</td> <td class="space">G. Gasso, A. Pappaioannou, M. Spivak, and L. Bottou, {\it Batch and online learning algorithms for nonconvex Neyman-Pearson classification}, ACM Trans. Intelligent System Technol., 3 (2011), article 28.</td> </tr><tr> <td>[66]</td> <td class="space">E. G. Gladyshev, {\it On stochastic approximations}, Theory Probab. Appl., 10 (1965), pp. 275-278.</td> </tr><tr> <td>[67]</td> <td class="space">R. Glowinski and A. Marrocco, {\it Sur l’approximation, par elements finis déordre un, et la resolution, par penalisation-dualité, d’une classe de problems de Dirichlet non lineares}, Rev. Française Automat. Informat. Recherche Opérationalle Sér. Rouge Anal. Numér., 9 (1975), pp. 41-76. · <a href="/0368.65053" class="nowrap">Zbl 0368.65053</a></td> </tr><tr> <td>[68]</td> <td class="space">D. Goldfarb, {\it A family of variable metric updates derived by variational means}, Math. Comp., 24 (1970), pp. 23-26. · <a href="/0196.18002" class="nowrap">Zbl 0196.18002</a></td> </tr><tr> <td>[69]</td> <td class="space">G. H. Golub and C. F. Van Loan, {\it Matrix Computations}, 4th ed., Johns Hopkins University Press, Baltimore, MD, 2012. · <a href="/1268.65037" class="nowrap">Zbl 1268.65037</a></td> </tr><tr> <td>[70]</td> <td class="space">I. J. Goodfellow, O. Vinyals, and A. M. Sake, {\it Qualitatively Characterizing Neural Network Optimization Problems}, CoRR preprint, , 2014.</td> </tr><tr> <td>[71]</td> <td class="space">A. Griewank, {\it Automatic differentiation}, in Princeton Companion to Applied Mathematics, N. Higham, ed., Princeton University Press, Princeton, NJ, 2014, pp. 749-752.</td> </tr><tr> <td>[72]</td> <td class="space">M. Gürbüzbalaban, A. Ozdaglar, and P. A. Parrilo, {\it On the convergence rate of incremental aggregated gradient algorithms}, SIAM J. Optim., 27 (2017), pp. 1035-1048, . · <a href="/1366.90195" class="nowrap">Zbl 1366.90195</a></td> </tr><tr> <td>[73]</td> <td class="space">F. S. Hashemi, S. Ghosh, and R. Pasupathy, {\it On adaptive sampling rules for stochastic recursions}, in Proceedings of the 2014 Winter Simulation Conference (WSC ’14), 2014, pp. 3959-3970.</td> </tr><tr> <td>[74]</td> <td class="space">K. He, X. Zhang, S. Ren, and J. Sun, {\it Deep residual learning for image recognition}, in Proc. IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2016, pp. 770-778.</td> </tr><tr> <td>[75]</td> <td class="space">W. Hoeffding, {\it Probability inequalities for sums of bounded random variables}, J. Amer. Statist. Assoc., 58 (1963), pp. 13-30. · <a href="/0127.10602" class="nowrap">Zbl 0127.10602</a></td> </tr><tr> <td>[76]</td> <td class="space">M. D. Hoffman, D. M. Blei, C. Wang, and J. W. Paisley, {\it Stochastic variational inference}, J. Mach. Learn.Res., 14 (2013), pp. 1303-1347. · <a href="/1317.68163" class="nowrap">Zbl 1317.68163</a></td> </tr><tr> <td>[77]</td> <td class="space">T. Homem-de Mello and A. Shapiro, {\it A simulation-based approach to two-stage stochastic programming with recourse}, Math. Program., 81 (1998), pp. 301-325. · <a href="/0919.90120" class="nowrap">Zbl 0919.90120</a></td> </tr><tr> <td>[78]</td> <td class="space">C.-J. Hsieh, M. A. Sustik, I. S. Dhillon, P. K. Ravikumar, and R. Poldrack, {\it BIG & QUIC: Sparse inverse covariance estimation for a million variables}, in Advances in Neural Information Processing Systems, 2013, pp. 3165-3173.</td> </tr><tr> <td>[79]</td> <td class="space">C. J. Hsieh, M. A. Sustik, P. Ravikumar, and I. S. Dhillon, {\it Sparse inverse covariance matrix estimation using quadratic approximation}, in Advances in Neural Information Processing Systems 24, 2011, pp. 2330-2338.</td> </tr><tr> <td>[80]</td> <td class="space">S. Ioffe and C. Szegedy, {\it Batch normalization: Accelerating deep network training by reducing internal covariate shift}, in Proceedings of the 32nd International Conference on Machine Learning (ICML 2015), Lille, France, 2015, pp. 448-456.</td> </tr><tr> <td>[81]</td> <td class="space">T. Joachims, {\it Text categorization with support vector machines: Learning with many relevant features}, in Proceedings of the 10th European Conference on Machine Learning (ECML’98), Chemnitz, Germany, Springer, 1998, pp. 137-142.</td> </tr><tr> <td>[82]</td> <td class="space">R. Johnson and T. Zhang, {\it Accelerating stochastic gradient descent using predictive variance reduction}, in Advances in Neural Information Processing Systems 26, 2013, pp. 315-323.</td> </tr><tr> <td>[83]</td> <td class="space">R. E. Kalman, {\it A new approach to linear filtering and prediction problems}, Trans. ASME J. Basic Engrg. Ser. D, 82 (1960), pp. 35-45.</td> </tr><tr> <td>[84]</td> <td class="space">D. P. Kingma and J. Ba, {\it Adam: A Method for Stochastic Optimization}, CoRR preprint, , 2014.</td> </tr><tr> <td>[85]</td> <td class="space">A. Krizhevsky, I. Sutskever, and G. E. Hinton, {\it ImageNet Classification with Deep Convolutional Neural Networks}, in Advances in Neural Information Processing Systems 25, 2012, pp. 84-90.</td> </tr><tr> <td>[86]</td> <td class="space">J. Lafond, N. Vasilache, and L. Bottou, {\it Manuscript in preparation}, 2016.</td> </tr><tr> <td>[87]</td> <td class="space">Y. Le Cun, B. E. Boser, J. S. Denker, D. Henderson, R. E. Howard, W. E. Hubbard, and L. D. Jackel, {\it Handwritten digit recognition with a back-propagation network}, in Advances in Neural Information Processing Systems 2, 1989, pp. 396-404.</td> </tr><tr> <td>[88]</td> <td class="space">Y. Le Cun, L. Bottou, Y. Bengio, and P. Haffner, {\it Gradient based learning applied to document recognition}, Proc. IEEE, 86 (1998), pp. 2278-2324.</td> </tr><tr> <td>[89]</td> <td class="space">Y. Le Cun, L. Bottou, G. B. Orr, and K.-R. Müller, {\it Efficient backprop}, in Neural Networks: Tricks of the Trade, Lecture Notes in Comput. Sci. 1524, Springer-Verlag, 1998, pp. 9-50.</td> </tr><tr> <td>[90]</td> <td class="space">N. Le Roux, M. Schmidt, and F. R. Bach, {\it A stochastic gradient method with an exponential convergence rate for finite training sets}, in Advances in Neural Information Processing Systems 25, 2012, pp. 2663-2671.</td> </tr><tr> <td>[91]</td> <td class="space">W. S. Lee, P. L. Bartlett, and R. C. Williamson, {\it The importance of convexity in learning with squared loss}, IEEE Trans. Inform. Theory, 44 (1998), pp. 1974-1980. · <a href="/0935.68091" class="nowrap">Zbl 0935.68091</a></td> </tr><tr> <td>[92]</td> <td class="space">T. K. Leen and G. B. Orr, {\it Optimal stochastic search and adaptive momentum}, in Advances in Neural Information Processing Systems 6, 1993, pp. 477-484.</td> </tr><tr> <td>[93]</td> <td class="space">D. Leventhal and A. S. Lewis, {\it Randomized methods for linear constraints: Convergence rates and conditioning}, Math. Oper. Res., 35 (2010), pp. 641-654. · <a href="/1216.15006" class="nowrap">Zbl 1216.15006</a></td> </tr><tr> <td>[94]</td> <td class="space">D. D. Lewis, Y. Yang, T. G. Rose, and F. Li, {\it RCV1: A new benchmark collection of text categorization research}, J. Mach. Learn. Res., 5 (2004), pp. 361-397.</td> </tr><tr> <td>[95]</td> <td class="space">C.-J. Lin and J. J. Moré, {\it Newton’s method for large bound-constrained optimization problems}, SIAM J. Optim., 9 (1999), pp. 1100-1127, . · <a href="/0957.65064" class="nowrap">Zbl 0957.65064</a></td> </tr><tr> <td>[96]</td> <td class="space">H. Lin, J. Mairal, and Z. Harchaoui, {\it A universal catalyst for first-order optimization}, Advances in Neural Information Processing Systems 28, 2015, pp. 3384-3392.</td> </tr><tr> <td>[97]</td> <td class="space">D. C. Liu and J. Nocedal, {\it On the limited memory BFGS method for large scale optimization}, Math. Program., 45 (1989), pp. 503-528. · <a href="/0696.90048" class="nowrap">Zbl 0696.90048</a></td> </tr><tr> <td>[98]</td> <td class="space">J. Liu, S. J. Wright, C. Ré, V. Bittorf, and S. Sridhar, {\it An asynchronous parallel stochastic coordinate descent algorithm}, J. Mach. Learn.Res., 16 (2015), pp. 285-322. · <a href="/1337.68286" class="nowrap">Zbl 1337.68286</a></td> </tr><tr> <td>[99]</td> <td class="space">G. Marceau-Caron and Y. Ollivier, {\it Practical Riemannian Neural Networks}, CoRR preprint, , 2016. · <a href="/1428.82047" class="nowrap">Zbl 1428.82047</a></td> </tr><tr> <td>[100]</td> <td class="space">J. Martens, {\it New Insights and Perspectives on the Natural Gradient Method}, preprint, , 2014. · <a href="/07306852" class="nowrap">Zbl 07306852</a></td> </tr><tr> <td>[101]</td> <td class="space">J. Martens and I. Sutskever, {\it Learning recurrent neural networks with Hessian-free optimization}, in 28th International Conference on Machine Learning (ICML), 2011.</td> </tr><tr> <td>[102]</td> <td class="space">P. Massart, {\it Some applications of concentration inequalities to statistics}, Ann. Fac. Sci. Toulouse Math. (6), 9 (2000), pp. 245-303. · <a href="/0986.62002" class="nowrap">Zbl 0986.62002</a></td> </tr><tr> <td>[103]</td> <td class="space">A. Mokhtari and A. Ribeiro, {\it RES: Regularized stochastic BFGS algorithm}, IEEE Trans. Signal Process., 62 (2014), pp. 6089-6104. · <a href="/1394.94405" class="nowrap">Zbl 1394.94405</a></td> </tr><tr> <td>[104]</td> <td class="space">N. Murata, {\it A statistical study of on-line learning}, in On-line Learning in Neural Networks, D. Saad, ed., Cambridge University Press, New York, 1998, pp. 63-92. · <a href="/0966.68170" class="nowrap">Zbl 0966.68170</a></td> </tr><tr> <td>[105]</td> <td class="space">A. Nemirovski, A. Juditsky, G. Lan, and A. Shapiro, {\it Robust stochastic approximation approach to stochastic programming}, SIAM J. Optim., 19 (2009), pp. 1574-1609, . · <a href="/1189.90109" class="nowrap">Zbl 1189.90109</a></td> </tr><tr> <td>[106]</td> <td class="space">A. S. Nemirovski and D. B. Yudin, {\it On Cesaro’s convergence of the steepest descent method for approximating saddle point of convex-concave functions}, Soviet Math. Dokl., 19 (1978). · <a href="/0413.90061" class="nowrap">Zbl 0413.90061</a></td> </tr><tr> <td>[107]</td> <td class="space">Y. Nesterov, {\it A method of solving a convex programming problem with convergence rate ${\mathcal O}(1/k^2)$}, Soviet Math. Dokl., 27 (1983), pp. 372-376. · <a href="/0535.90071" class="nowrap">Zbl 0535.90071</a></td> </tr><tr> <td>[108]</td> <td class="space">Y. Nesterov, {\it Introductory Lectures on Convex Optimization: A Basic Course}, Kluwer Academic, 2004. · <a href="/1086.90045" class="nowrap">Zbl 1086.90045</a></td> </tr><tr> <td>[109]</td> <td class="space">Y. Nesterov, {\it Primal-dual subgradient methods for convex problems}, Math. Program. Ser. B, 120 (2009), pp. 221-259. · <a href="/1191.90038" class="nowrap">Zbl 1191.90038</a></td> </tr><tr> <td>[110]</td> <td class="space">Y. Nesterov, {\it Efficiency of coordinate descent methods on huge-scale optimization problems}, SIAM J. Optim., 22 (2012), pp. 341-362, . · <a href="/1257.90073" class="nowrap">Zbl 1257.90073</a></td> </tr><tr> <td>[111]</td> <td class="space">NIPS Foundation, {\it Advances in Neural Information Processing Systems $(29$ volumes)}, 1987-2016. Volumes 0-28, .</td> </tr><tr> <td>[112]</td> <td class="space">F. Niu, B. Recht, C. Re, and S. J. Wright, {\it Hogwild!: A lock-free approach to parallelizing stochastic gradient descent}, in Advances in Neural Information Processing Systems 24, 2011, pp. 693-701.</td> </tr><tr> <td>[113]</td> <td class="space">J. Nocedal, {\it Updating quasi-Newton matrices with limited storage}, Math. Comp., 35 (1980), pp. 773-782. · <a href="/0464.65037" class="nowrap">Zbl 0464.65037</a></td> </tr><tr> <td>[114]</td> <td class="space">J. Nocedal and S. J. Wright, {\it Numerical Optimization}, 2nd ed., Springer, New York, 2006. · <a href="/1104.65059" class="nowrap">Zbl 1104.65059</a></td> </tr><tr> <td>[115]</td> <td class="space">A. B. J. Novikoff, {\it On convergence proofs on perceptrons}, in Proceedings of the Symposium on the Mathematical Theory of Automata, Vol. 12, 1962, pp. 615-622.</td> </tr><tr> <td>[116]</td> <td class="space">J. Nutini, M. Schmidt, I. H. Laradji, M. Friedlander, and H. Koepke, {\it Coordinate descent converges faster with the Gauss-Southwell rule than random selection}, in 32nd International Conference on Machine Learning (ICML), 2015, pp. 1632-1641.</td> </tr><tr> <td>[117]</td> <td class="space">J. M. Ortega and W. C. Rheinboldt, {\it Iterative Solution of Nonlinear Equations in Several Variables}, SIAM, Philadelphia, 2000, . · <a href="/0949.65053" class="nowrap">Zbl 0949.65053</a></td> </tr><tr> <td>[118]</td> <td class="space">M. R. Osborne, {\it Fisher’s method of scoring}, Internat. Statist. Rev., 60 (1992), pp. 99-117. · <a href="/0755.62005" class="nowrap">Zbl 0755.62005</a></td> </tr><tr> <td>[119]</td> <td class="space">H. Park, S. Amari, and K. Fukumizu, {\it Adaptive natural gradient learning algorithms for various stochastic models}, Neural Networks, 13 (2000), pp. 755-764.</td> </tr><tr> <td>[120]</td> <td class="space">R. Pasupathy, P. W. Glynn, S. Ghosh, and F. S. Hashemi, {\it On sampling rates in stochastic recursions}, submitted for publication (2015). · <a href="/1380.93168" class="nowrap">Zbl 1380.93168</a></td> </tr><tr> <td>[121]</td> <td class="space">B. A. Pearlmutter, {\it Fast exact multiplication by the Hessian}, Neural Comput., 6 (1994), pp. 147-160.</td> </tr><tr> <td>[122]</td> <td class="space">M. Pilanci and M. J. Wainwright, {\it Newton sketch: A linear-time optimization algorithm with linear-quadratic convergence}, SIAM J. Optim., 27 (2017), pp. 205-245, . · <a href="/1456.90125" class="nowrap">Zbl 1456.90125</a></td> </tr><tr> <td>[123]</td> <td class="space">B. T. Polyak, {\it Some methods of speeding up the convergence of iteration methods}, USSR Comput. Math. Math. Phys., 4 (1964), pp. 1-17. · <a href="/0147.35301" class="nowrap">Zbl 0147.35301</a></td> </tr><tr> <td>[124]</td> <td class="space">B. T. Polyak, {\it Comparison of the convergence rates for single-step and multi-step optimization algorithms in the presence of noise}, Engrg. Cybernet., 15 (1977), pp. 6-10.</td> </tr><tr> <td>[125]</td> <td class="space">B. T. Polyak, {\it New method of stochastic approximation type}, Automat. Remote Control, 51 (1991), pp. 937-946. · <a href="/0737.93080" class="nowrap">Zbl 0737.93080</a></td> </tr><tr> <td>[126]</td> <td class="space">B. T. Polyak and A. B. Juditsky, {\it Acceleration of stochastic approximation by averaging}, SIAM J. Control Optim., 30 (1992), pp. 838-855, . · <a href="/0762.62022" class="nowrap">Zbl 0762.62022</a></td> </tr><tr> <td>[127]</td> <td class="space">M. J. D. Powell, {\it On search directions for minimization algorithms}, Math. Program., 4 (1973), pp. 193-201. · <a href="/0258.90043" class="nowrap">Zbl 0258.90043</a></td> </tr><tr> <td>[128]</td> <td class="space">M. Raginsky and A. Rakhlin, {\it Information-based complexity, feedback and dynamics in convex programming}, IEEE Trans. Inform. Theory, 57 (2011), pp. 7036-7056. · <a href="/1365.93191" class="nowrap">Zbl 1365.93191</a></td> </tr><tr> <td>[129]</td> <td class="space">A. Razavian, H. Azizpour, J. Sullivan, and S. Carlsson, {\it CNN features off-the-shelf: An astounding baseline for recognition}, in 2014 IEEE Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2014, pp. 512-519.</td> </tr><tr> <td>[130]</td> <td class="space">H. Robbins and S. Monro, {\it A stochastic approximation method}, Ann. Math. Statist., 22 (1951), pp. 400-407. · <a href="/0054.05901" class="nowrap">Zbl 0054.05901</a></td> </tr><tr> <td>[131]</td> <td class="space">H. Robbins and D. Siegmund, {\it A convergence theorem for non negative almost supermartingales and some applications}, in Optimizing Methods in Statistics, J. S. Rustagi, ed., Academic Press, 1971, pp. 233-257. · <a href="/0286.60025" class="nowrap">Zbl 0286.60025</a></td> </tr><tr> <td>[132]</td> <td class="space">F. Roosta-Khorasani and M. W. Mahoney, {\it Sub-sampled Newton Methods II: Local Convergence Rates}, preprint, , 2016. · <a href="/1412.49059" class="nowrap">Zbl 1412.49059</a></td> </tr><tr> <td>[133]</td> <td class="space">S. Ross, P. Mineiro, and J. Langford, {\it Normalized online learning}, in Proceedings of the Twenty-Ninth Conference on Uncertainty in Artificial Intelligence (UAI), 2013.</td> </tr><tr> <td>[134]</td> <td class="space">D. E. Rumelhart, G. E. Hinton, and R. J. Williams, {\it Learning internal representations by error propagation}, in Parallel Distributed Processing: Explorations in the Microstructure of Cognition, Vol. 1, MIT Press, Cambridge, MA, 1986, pp. 318-362.</td> </tr><tr> <td>[135]</td> <td class="space">D. E. Rumelhart, G. E. Hinton, and R. J. Williams, {\it Learning representations by back-propagating errors}, Nature, 323 (1986), pp. 533-536. · <a href="/1369.68284" class="nowrap">Zbl 1369.68284</a></td> </tr><tr> <td>[136]</td> <td class="space">D. Ruppert, {\it Efficient Estimations from a Slowly Convergent Robbins-Monro Process}, Tech. report 781, Cornell University Operations Research and Industrial Engineering, 1988.</td> </tr><tr> <td>[137]</td> <td class="space">K. Scheinberg and X. Tang, {\it Practical inexact proximal quasi-Newton method with global complexity analysis}, Math. Program., 160 (2016), pp. 495-529. · <a href="/1366.90166" class="nowrap">Zbl 1366.90166</a></td> </tr><tr> <td>[138]</td> <td class="space">M. Schmidt, {\it Graphical Model Structure Learning with $ℓ_1$-Regularization}, Ph.D. thesis, University of British Columbia, 2010.</td> </tr><tr> <td>[139]</td> <td class="space">M. Schmidt, N. Le Roux, and F. Bach, {\it Minimizing finite sums with the stochastic average gradient}, Math. Program., 162 (2017), pp. 83-112. · <a href="/1358.90073" class="nowrap">Zbl 1358.90073</a></td> </tr><tr> <td>[140]</td> <td class="space">M. Schmidt, N. L. Roux, and F. R. Bach, {\it Convergence rates of inexact proximal-gradient methods for convex optimization}, in Advances in Neural Information Processing Systems 24, 2011, pp. 1458-1466.</td> </tr><tr> <td>[141]</td> <td class="space">N. N. Schraudolph, {\it Fast curvature matrix-vector products}, in Artificial Neural Networks, ICANN 2001, G. Dorffner, H. Bischof, and K. Hornik, eds., Lecture Notes in Comput. Sci. 2130, Springer-Verlag, Berlin, Heidelberg, 2001, pp. 19-26. · <a href="/1001.68684" class="nowrap">Zbl 1001.68684</a></td> </tr><tr> <td>[142]</td> <td class="space">N. N. Schraudolph, J. Yu, and S. Günter, {\it A stochastic quasi-Newton method for online convex optimization}, in International Conference on Artificial Intelligence and Statistics, Society for Artificial Intelligence and Statistics, 2007, pp. 436-443.</td> </tr><tr> <td>[143]</td> <td class="space">G. Shafer and V. Vovk, {\it Probability and Finance: It’s Only a Game!}, John Wiley & Sons, 2005. · <a href="/0985.91024" class="nowrap">Zbl 0985.91024</a></td> </tr><tr> <td>[144]</td> <td class="space">S. Shalev-Shwartz, Y. Singer, N. Srebro, and A. Cotter, {\it Pegasos: Primal estimated sub-gradient solver for SVM}, Math. Program., 127 (2011), pp. 3-30. · <a href="/1211.90239" class="nowrap">Zbl 1211.90239</a></td> </tr><tr> <td>[145]</td> <td class="space">S. Shalev-Shwartz and T. Zhang, {\it Stochastic dual coordinate ascent methods for regularized loss}, J. Mach. Learn. Res., 14 (2013), pp. 567-599. · <a href="/1307.68073" class="nowrap">Zbl 1307.68073</a></td> </tr><tr> <td>[146]</td> <td class="space">D. F. Shanno, {\it Conditioning of quasi-Newton methods for function minimization}, Math. Comp., 24 (1970), pp. 647-656. · <a href="/0225.65073" class="nowrap">Zbl 0225.65073</a></td> </tr><tr> <td>[147]</td> <td class="space">S. Sra, S. Nowozin, and S. Wright, {\it Optimization for Machine Learning}, MIT Press, Cambridge, MA, 2011.</td> </tr><tr> <td>[148]</td> <td class="space">Stanford Vision Lab, {\it ImageNet Large Scale Visual Recognition Challenge (ILSVRC)}, , 2015 (accessed October 25, 2015).</td> </tr><tr> <td>[149]</td> <td class="space">T. Steihaug, {\it The conjugate gradient method and trust regions in large scale optimization}, SIAM J. Numer. Anal., 20 (1983), pp. 626-637, . · <a href="/0518.65042" class="nowrap">Zbl 0518.65042</a></td> </tr><tr> <td>[150]</td> <td class="space">I. Sutskever, J. Martens, G. Dahl, and G. Hinton, {\it On the importance of initialization and momentum in deep learning}, in 30th International Conference on Machine Learning (ICML), 2013.</td> </tr><tr> <td>[151]</td> <td class="space">R. Tibshirani, {\it Regression shrinkage and selection via the lasso}, J. Roy. Statist. Soc. Ser. B, 58 (1996), pp. 267-288. · <a href="/0850.62538" class="nowrap">Zbl 0850.62538</a></td> </tr><tr> <td>[152]</td> <td class="space">T. Tieleman and G. Hinton, {\it Lecture 6.5. RMSPROP: Divide the gradient by a running average of its recent magnitude.} COURSERA: Neural Networks for Machine Learning, 2012.</td> </tr><tr> <td>[153]</td> <td class="space">P. Tseng and S. Yun, {\it A coordinate gradient descent method for nonsmooth separable minimization}, Math. Program., 117 (2009), pp. 387-423. · <a href="/1166.90016" class="nowrap">Zbl 1166.90016</a></td> </tr><tr> <td>[154]</td> <td class="space">A. B. Tsybakov, {\it Optimal aggregation of classifiers in statistical learning}, Ann. Statist., 32 (2004), pp. 135-166. · <a href="/1105.62353" class="nowrap">Zbl 1105.62353</a></td> </tr><tr> <td>[155]</td> <td class="space">V. N. Vapnik, {\it Estimation of Dependences Based on Empirical Data}, Springer, 1983. · <a href="/0499.62005" class="nowrap">Zbl 0499.62005</a></td> </tr><tr> <td>[156]</td> <td class="space">V. N. Vapnik, {\it Statistical Learning Theory}, Wiley, New York, 1998. · <a href="/0935.62007" class="nowrap">Zbl 0935.62007</a></td> </tr><tr> <td>[157]</td> <td class="space">V. N. Vapnik and A. Y. Chervonenkis, {\it On the uniform convergence of relative frequencies of events to their probabilities}, Proc. USSR Acad. Sci., 181 (1968), pp. 781-783; English translation: Soviet Math. Dokl., 9 (1968), pp. 915-918. · <a href="/0247.60004" class="nowrap">Zbl 0247.60004</a></td> </tr><tr> <td>[158]</td> <td class="space">V. N. Vapnik and A. Y. Chervonenkis, {\it Theory of Pattern Recognition}, Nauka, Moscow, 1974; German Translation: {\it Theorie der Zeichenerkennung}, Akademie-Verlag, Berlin, 1979. · <a href="/0284.68070" class="nowrap">Zbl 0284.68070</a></td> </tr><tr> <td>[159]</td> <td class="space">S. J. Wright, R. D. Nowak, and M. A. T. Figueiredo, {\it Sparse reconstruction by separable approximation}, IEEE Trans. Signal Process., 57 (2009), pp. 2479-2493. · <a href="/1391.94442" class="nowrap">Zbl 1391.94442</a></td> </tr><tr> <td>[160]</td> <td class="space">C. F. J. Wu, {\it On the convergence properties of the EM algorithm}, Ann. Statist., 11 (1983), pp. 95-103, . · <a href="/0517.62035" class="nowrap">Zbl 0517.62035</a></td> </tr><tr> <td>[161]</td> <td class="space">W. Xu, {\it Towards Optimal One Pass Large Scale Learning with Averaged Stochastic Gradient Descent}, preprint, , 2011.</td> </tr><tr> <td>[162]</td> <td class="space">M. D. Zeiler, {\it ADADELTA: An Adaptive Learning Rate Method}, CoRR preprint, , 2012.</td> </tr><tr> <td>[163]</td> <td class="space">C. Zhu, R. H. Byrd, P. Lu, and J. Nocedal, {\it Algorithm 78: L-BFGS-B: Fortran subroutines for large-scale bound constrained optimization}, ACM Trans. Math. Softw., 23 (1997), pp. 550-560. · <a href="/0912.65057" class="nowrap">Zbl 0912.65057</a></td> </tr><tr> <td>[164]</td> <td class="space">M. Zinkevich, {\it Online Convex Programming and Generalized Infinitesimal Gradient Ascent}, in Proceedings of the 20th International Conference on Machine Learning (ICML), 2003, pp. 928-936.</td> </tr></table> <div class="reference_disclaimer"> This reference list is based on information provided by the publisher or from digital mathematics libraries. Its items are heuristically matched to zbMATH identifiers and may contain data conversion errors. In some cases that data have been complemented/enhanced by data from zbMATH Open. This attempts to reflect the references listed in the original paper as accurately as possible without claiming completeness or a perfect matching. </div> </div></article> </div></div> </div> </div> <div class="clearfix"></div> </div> </div> <div id="foot"><div class="copyright"> © 2025 <a target="fiz" href="https://www.fiz-karlsruhe.de/en">FIZ Karlsruhe GmbH</a> <a href="/privacy-policy/">Privacy Policy</a> <a href="/legal-notices/">Legal Notices</a> <a href="/terms-conditions/">Terms & Conditions</a> <div class="info"> <ul class="nav"> <li class="mastodon"> <a href="https://mathstodon.xyz/@zbMATH" target="_blank" class="no-new-tab-icon"> <img src="/static/mastodon.png" title="zbMATH at Mathstodon (opens in new tab)" alt="Mastodon logo"> </a> </li> </ul> </div> </div> <div class="clearfix" style="height: 0px;"></div> </div> </div> <script src="https://static.zbmath.org/contrib/jquery/1.9.1/jquery.min.js"></script> <script src="https://static.zbmath.org/contrib/jquery-caret/1.5.2/jquery.caret.min.js"></script> <script src="/static/js/jquery-ui-1.10.1.custom.min.js"></script> <script src="https://static.zbmath.org/contrib/bootstrap/v3.3.7zb1/js/bootstrap.min.js"></script> <script src="https://static.zbmath.org/contrib/bootstrap-lightbox/v0.7.0/bootstrap-lightbox.min.js"></script> <script src="https://static.zbmath.org/contrib/retina/unknown/retina.js"></script> <script src="https://static.zbmath.org/contrib/bootstrap-select/v1.13.14/js/bootstrap-select.min.js"></script> <script> var SCRIPT_ROOT = ""; </script> <script src="/static/scripts.js?v=20240926"> </script> <script src="https://static.zbmath.org/contrib/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> <script type="text/x-mathjax-config"> MathJax.Hub.Config({ "HTML-CSS": { preferredFont: "TeX", availableFonts: [ "STIX", "TeX" ], linebreaks: { automatic: true }, EqnChunk: (MathJax.Hub.Browser.isMobile ? 10 : 50) }, tex2jax: { processEscapes: true, ignoreClass: "tex2jax_ignore|dno" }, TeX: { Macros: { Aut: "\\operatorname{Aut}", Hom: "\\operatorname{Hom}" }, noUndefined: { attributes: { mathcolor: "#039", //"red", mathbackground: "white", //"#FFEEEE", mathsize: "90%" } } }, messageStyle: "none" }); </script> <script type="text/javascript"> $(document).ready(function() { $("#MathInput").stop(true, true).keyup(function() { $.ajax({ url: "/mwsq/", type: "POST", data: { query : $("#MathInput").val() }, dataType: "text" }) .done(function(xml) { $("#MathPreview").html(xml); $(window).resize(); }); }); var press = jQuery.Event("keyup"); press.ctrlKey = false; press.which = 40; $("#MathInput").trigger(press); }); </script> <div id="new_tab_icon" style="display: none"> <span class="glyphicon glyphicon-new-window" aria-hidden="true"></span><span class="sr-only">(opens in new tab)</span></div> </body> </html>

CINXE.COM

Document Zbl 1397.65085 - zbMATH Open