CINXE.COM

<!DOCTYPE html> <html lang="en" dir="ltr" class="client-nojs"> <head> <title>Deep belief networks - Scholarpedia</title> <meta charset="UTF-8" /> <meta name="generator" content="MediaWiki 1.19.17" /> <meta name="citation_title" content="Deep belief networks" /> <meta name="citation_author" content="Geoffrey E. Hinton" /> <meta name="citation_date" content="2009/5/31" /> <meta name="citation_journal_title" content="Scholarpedia" /> <meta name="citation_issn" content="1941-6016" /> <meta name="citation_volume" content="4" /> <meta name="citation_issue" content="5" /> <meta name="citation_firstpage" content="5947" /> <meta name="citation_doi" content="10.4249/scholarpedia.5947" /> <link rel="shortcut icon" href="/w/images/6/64/Favicon.ico" /> <link rel="search" type="application/opensearchdescription+xml" href="/w/opensearch_desc.php" title="Scholarpedia (en)" /> <link rel="EditURI" type="application/rsd+xml" href="http://www.scholarpedia.org/w/api.php?action=rsd" /> <link rel="alternate" type="application/atom+xml" title="Scholarpedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom" /> <link rel="stylesheet" href="http://www.scholarpedia.org/w/load.php?debug=false&lang=en&modules=mediawiki.legacy.commonPrint%2Cshared%7Cskins.vector&only=styles&skin=vector&*" /> <link rel="stylesheet" href="/w/skins/vector/font-awesome.min.css" /> <link rel="stylesheet" href="/w/skins/vector/local-screen.css" /><meta name="ResourceLoaderDynamicStyles" content="" /> <link rel="stylesheet" href="http://www.scholarpedia.org/w/load.php?debug=false&lang=en&modules=site&only=styles&skin=vector&*" /> <style>a:lang(ar),a:lang(ckb),a:lang(fa),a:lang(kk-arab),a:lang(mzn),a:lang(ps),a:lang(ur){text-decoration:none}a.new,#quickbar a.new{color:#ba0000} /* cache key: wikidb:resourceloader:filter:minify-css:7:c88e2bcd56513749bec09a7e29cb3ffa */</style> <script src="http://www.scholarpedia.org/w/load.php?debug=false&lang=en&modules=startup&only=scripts&skin=vector&*"></script> <script>if(window.mw){ mw.config.set({"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Deep_belief_networks","wgTitle":"Deep belief networks","wgCurRevisionId":91189,"wgArticleId":5947,"wgIsArticle":true,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Pattern Recognition"],"wgBreakFrames":false,"wgPageContentLanguage":"en","wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgRelevantPageName":"Deep_belief_networks","wgRestrictionEdit":[],"wgRestrictionMove":[],"wgVectorEnabledModules":{"collapsiblenav":true,"collapsibletabs":true,"editwarning":false,"expandablesearch":false,"footercleanup":false,"sectioneditlinks":false,"simplesearch":true,"experiments":true}}); }</script><script>if(window.mw){ mw.loader.implement("user.options",function($){mw.user.options.set({"ccmeonemails":0,"cols":80,"date":"default","diffonly":0,"disablemail":0,"disablesuggest":0,"editfont":"default","editondblclick":0,"editsection":1,"editsectiononrightclick":0,"enotifminoredits":0,"enotifrevealaddr":0,"enotifusertalkpages":1,"enotifwatchlistpages":1,"extendwatchlist":0,"externaldiff":0,"externaleditor":0,"fancysig":0,"forceeditsummary":0,"gender":"unknown","hideminor":0,"hidepatrolled":0,"highlightbroken":1,"imagesize":2,"justify":0,"math":1,"minordefault":0,"newpageshidepatrolled":0,"nocache":0,"noconvertlink":0,"norollbackdiff":0,"numberheadings":0,"previewonfirst":0,"previewontop":1,"quickbar":5,"rcdays":7,"rclimit":50,"rememberpassword":0,"rows":25,"searchlimit":20,"showhiddencats":0,"showjumplinks":1,"shownumberswatching":1,"showtoc":1,"showtoolbar":1,"skin":"vector","stubthreshold":0,"thumbsize":2,"underline":2,"uselivepreview":0,"usenewrc":0,"watchcreations":0,"watchdefault":0,"watchdeletion":0, "watchlistdays":3,"watchlisthideanons":0,"watchlisthidebots":0,"watchlisthideliu":0,"watchlisthideminor":0,"watchlisthideown":0,"watchlisthidepatrolled":0,"watchmoves":0,"wllimit":250,"vector-simplesearch":1,"vector-collapsiblenav":1,"variant":"en","language":"en","searchNs0":true,"searchNs1":false,"searchNs2":false,"searchNs3":false,"searchNs4":false,"searchNs5":false,"searchNs6":false,"searchNs7":false,"searchNs8":false,"searchNs9":false,"searchNs10":false,"searchNs11":false,"searchNs12":false,"searchNs13":false,"searchNs14":false,"searchNs15":false,"searchNs200":false,"searchNs201":false,"searchNs400":false,"searchNs401":false});;},{},{});mw.loader.implement("user.tokens",function($){mw.user.tokens.set({"editToken":"+\\","watchToken":false});;},{},{}); /* cache key: wikidb:resourceloader:filter:minify-js:7:e87579b4b142a5fce16144e6d8ce1889 */ }</script> <script>if(window.mw){ mw.loader.load(["mediawiki.page.startup","mediawiki.legacy.wikibits","mediawiki.legacy.ajax"]); }</script> <link rel="canonical" href="http://www.scholarpedia.org/article/Deep_belief_networks" /> </head> <body class="mediawiki ltr sitedir-ltr ns-0 ns-subject page-Deep_belief_networks skin-vector action-view cp-body-published"> <div id="mw-page-base" class="noprint"></div> <div id="mw-head-base" class="noprint"></div>  <div id="content" class="mw-body"> <a id="top"></a> <div id="mw-js-message" style="display:none;"></div>  <div id="siteNotice"><script type="text/javascript"> /* <![CDATA[ */ document.writeln("\x3cdiv id=\"localNotice\" lang=\"en\" dir=\"ltr\"\x3e\x3cp style=text-align:left;font-style:italic\x3eScholarpedia is supported by \x3ca href=\'http://www.braincorp.com\'\x3eBrain Corporation\x3c/a\x3e\x3c/p\x3e\x3c/div\x3e"); /* ]]> */ </script></div>   <h1 id="firstHeading" class="firstHeading"> <span dir="auto">Deep belief networks</span> </h1>  <div class="cp-googleplus"> <div class="g-plusone" align="right" data-size="small" data-annotation="inline" data-width="180"></div> <script type="text/javascript"> (function () { var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true; po.src = 'https://apis.google.com/js/plusone.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s); })(); </script> </div>  <div id="bodyContent">  <div id="siteSub">From Scholarpedia</div>   <div id="contentSub"><span class="subpages"><table class="cp-citation-subtitle" width="100%" cellpadding="0" cellspacing="0" border="0"> <tr valign="bottom"> <td align="left">Geoffrey E. Hinton (2009), Scholarpedia, 4(5):5947.</td> <td align="center"><a href="http://dx.doi.org/10.4249/scholarpedia.5947">doi:10.4249/scholarpedia.5947</a></td> <td align="right">revision #91189 [<a href="/w/index.php?title=Deep_belief_networks&action=cite&rev=91189" title="Deep belief networks">link to/cite this article</a>]</td> </tr> </table> </span></div>   <div id="jump-to-nav" class="mw-jump"> Jump to: <a href="#mw-head">navigation</a>, <a href="#p-search">search</a> </div>   <div id="mw-content-text" lang="en" dir="ltr" class="mw-content-ltr"><div class="cp-box-container"><div class="cp-curator-box noprint"><b><u>Post-publication activity</u></b><br /><button class="cp-button btn"></button><p><span class="cp-title-label">Curator:</span> <a href="/article/User:Geoffrey_E._Hinton" title="User:Geoffrey E. Hinton">Geoffrey E. Hinton</a> </p><div class="cp-assistants hidden"><div><span class="cp-title-label">Contributors:</span><p> </p></div><div><span></span><p><a href="/article/User:Ke_CHEN" title="User:Ke CHEN">Ke CHEN</a> </p></div><div><span></span><p><a href="/article/User:Eugene_M._Izhikevich" title="User:Eugene M. Izhikevich">Eugene M. Izhikevich</a> </p></div><div><span></span><p><a href="/article/User:Yoshua_Bengio" title="User:Yoshua Bengio">Yoshua Bengio</a> </p></div><div><span></span><p><a href="/article/User:Max_Welling" title="User:Max Welling">Max Welling</a> </p></div></div></div></div><div class="cp-author-order"><ul id="sp_authors"><li id="sort-1"><p><a href="/article/User:Geoffrey_E._Hinton" title="User:Geoffrey E. Hinton"><span class="bold">Dr. Geoffrey E. Hinton</span>, University of Toronto, CANADA</a> </p></li></ul></div><p><strong><span class="tex2jax_ignore">Deep belief nets</span></strong> are probabilistic generative models that are composed of multiple layers of stochastic, latent variables. The latent variables typically have binary values and are often called <i>hidden units</i> or <i>feature detectors</i>. The top two layers have undirected, symmetric connections between them and form an associative <a href="/article/Memory" title="Memory">memory</a>. The lower layers receive top-down, directed connections from the layer above. The states of the units in the lowest layer represent a data vector. </p><p>The two most significant properties of deep belief nets are: </p> <ul><li>There is an efficient, layer-by-layer procedure for learning the top-down, generative weights that determine how the variables in one layer depend on the variables in the layer above. </li></ul> <ul><li>After learning, the values of the latent variables in every layer can be inferred by a single, bottom-up pass that starts with an observed data vector in the bottom layer and uses the generative weights in the reverse direction. </li></ul> <p>Deep belief nets are learned one layer at a time by treating the values of the latent variables in one layer, when they are being inferred from data, as the data for training the next layer. This efficient, greedy learning can be followed by, or combined with, other learning procedures that fine-tune all of the weights to improve the generative or discriminative performance of the whole network. </p><p>Discriminative fine-tuning can be performed by adding a final layer of variables that represent the desired outputs and backpropagating error derivatives. When networks with many hidden layers are applied to highly-structured input data, such as images, backpropagation works much better if the feature detectors in the hidden layers are initialized by learning a deep belief net that models the structure in the input data (Hinton & Salakhutdinov, 2006). </p> <table id="toc" class="toc"><tr><td><div id="toctitle"><h2>Contents</h2></div> <ul> <li class="toclevel-1 tocsection-1"><a href="#Deep_Belief_Nets_as_Compositions_of_Simple_Learning_Modules"><span class="tocnumber">1</span> <span class="toctext">Deep Belief Nets as Compositions of Simple Learning Modules</span></a></li> <li class="toclevel-1 tocsection-2"><a href="#The_Theoretical_Justification_of_the_Learning_Procedure"><span class="tocnumber">2</span> <span class="toctext">The Theoretical Justification of the Learning Procedure</span></a></li> <li class="toclevel-1 tocsection-3"><a href="#Deep_Belief_Nets_with_Other_Types_of_Variable"><span class="tocnumber">3</span> <span class="toctext">Deep Belief Nets with Other Types of Variable</span></a></li> <li class="toclevel-1 tocsection-4"><a href="#Using_Autoencoders_as_the_Learning_Module"><span class="tocnumber">4</span> <span class="toctext">Using Autoencoders as the Learning Module</span></a></li> <li class="toclevel-1 tocsection-5"><a href="#Applications_of_Deep_Belief_Nets"><span class="tocnumber">5</span> <span class="toctext">Applications of Deep Belief Nets</span></a></li> <li class="toclevel-1 tocsection-6"><a href="#References"><span class="tocnumber">6</span> <span class="toctext">References</span></a></li> <li class="toclevel-1 tocsection-7"><a href="#See_also"><span class="tocnumber">7</span> <span class="toctext">See also</span></a></li> </ul> </td></tr></table> <h2> <span class="mw-headline" id="Deep_Belief_Nets_as_Compositions_of_Simple_Learning_Modules"> Deep Belief Nets as Compositions of Simple Learning Modules </span></h2> <p>A deep belief net can be viewed as a composition of simple learning modules each of which is a restricted type of <a href="/article/Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a> that contains a layer of <i>visible units</i> that represent the data and a layer of <i>hidden units</i> that learn to represent features that capture higher-order correlations in the data. The two layers are connected by a matrix of symmetrically weighted connections, $W\ ,$ and there are no connections within a layer. Given a vector of activities $v$ for the visible units, the hidden units are all conditionally independent so it is easy to sample a vector, $h\ ,$ from the factorial posterior distribution over hidden vectors, $p(h|v,W)\ .$ It is also easy to sample from $p(v|h,W)\ .$ By starting with an observed data vector on the visible units and alternating several times between sampling from $p(h|v,W)$ and $p(v| h,W)\ ,$ it is easy to get a learning signal. This signal is simply the difference between the pairwise correlations of the visible and hidden units at the beginning and end of the sampling (see Boltzmann machine for details). </p> <h2> <span class="mw-headline" id="The_Theoretical_Justification_of_the_Learning_Procedure">The Theoretical Justification of the Learning Procedure</span></h2> <p>The key idea behind deep belief nets is that the weights, $W\ ,$ learned by a restricted Boltzmann machine define both $p(v|h,W)$ and the prior distribution over hidden vectors, $p(h|W)\ ,$ so the probability of generating a visible vector, $v\ ,$ can be written as: \[ p(v) = \sum_h p(h|W)p(v|h,W) \] After learning $W\ ,$ we keep $p(v|h,W)$ but we replace $p(h|W)$ by a better model of the <i>aggregated</i> posterior distribution over hidden vectors – i.e. the non-factorial distribution produced by <a href="/article/Averaging" title="Averaging">averaging</a> the factorial posterior distributions produced by the individual data vectors. The better model is learned by treating the hidden activity vectors produced from the training data as the training data for the next learning module. Hinton, Osindero and Teh (2006) show that this replacement, if performed in the right way, improves a variational lower bound on the probability of the training data under the composite model. </p> <h2> <span class="mw-headline" id="Deep_Belief_Nets_with_Other_Types_of_Variable">Deep Belief Nets with Other Types of Variable</span></h2> <p>Deep belief nets typically use a logistic function of the weighted input received from above or below to determine the probability that a binary latent variable has a value of 1 during top-down generation or bottom-up inference, but other types of variable can be used (Welling et. al. 2005) and the variational bound still applies, provided the variables are all in the exponential family (i.e. the log probability is linear in the parameters). </p> <h2> <span class="mw-headline" id="Using_Autoencoders_as_the_Learning_Module">Using Autoencoders as the Learning Module</span></h2> <p>A closely related approach, that is also called a deep belief net,uses the same type of greedy, layer-by-layer learning with a different kind of learning module -- an <i>autoencoder</i> that simply tries to reproduce each data vector from the feature activations that it causes (Bengio et.al., 2007; LeCun et. al. 2007). However, the variational bound no longer applies and an autoencoder module is less good at ignoring random noise in its training data (Larochelle et.al., 2007). </p> <h2> <span class="mw-headline" id="Applications_of_Deep_Belief_Nets">Applications of Deep Belief Nets</span></h2> <p>Deep belief nets have been used for generating and recognizing images (Hinton, Osindero & Teh 2006, Ranzato et. al. 2007, Bengio et.al., 2007), video sequences (Sutskever and Hinton, 2007), and motion-capture data (Taylor et. al. 2007). If the number of units in the highest layer is small, deep belief nets perform non-linear <a href="/article/Dimensionality_reduction" title="Dimensionality reduction" class="cpstub">dimensionality reduction</a> and they can learn short binary codes that allow very fast retrieval of documents or images (Hinton & Salakhutdinov,2006; Salakhutdinov and Hinton,2007). </p> <h2> <span class="mw-headline" id="References">References</span></h2> <ul><li>Bengio, Y., Lamblin, P., Popovici, P., Larochelle, H. (2007) Greedy Layer-Wise Training of Deep Networks, Advances in <a href="/article/Neuron" title="Neuron">Neural</a> Information Processing Systems 19, MIT Press, Cambridge, MA. </li></ul> <ul><li> Hinton, G. E, Osindero, S., and Teh, Y. W. (2006). A fast learning <a href="/article/Algorithm" title="Algorithm">algorithm</a> for deep belief nets. Neural Computation, 18:1527-1554. </li></ul> <ul><li> Hinton, G. E. and Salakhutdinov, R. R. (2006). Reducing the dimensionality of data with neural networks. Science, 313:504-507. </li></ul> <ul><li> Larochelle, H., Erhan, D., Courville, A., Bergstra, J., Bengio, Y. (2007) An Empirical Evaluation of Deep Architectures on Problems with Many Factors of Variation. International Conference on Machine Learning. </li></ul> <ul><li> LeCun, Y. and Bengio, Y. (2007) Scaling Learning Algorithms Towards AI. In Bottou et al. (Eds.) Large-Scale Kernel Machines, MIT Press. </li></ul> <ul><li> M. Ranzato, F.J. Huang, Y. Boureau, Y. LeCun (2007) Unsupervised Learning of Invariant Feature Hierarchies with Applications to Object Recognition. Proc. of Computer <a href="/article/Vision" title="Vision">Vision</a> and Pattern Recognition Conference (CVPR 2007), Minneapolis, Minnesota, 2007 </li></ul> <ul><li> Salakhutdinov, R. R. and Hinton,G. E. (2007) Semantic Hashing. In Proceedings of the SIGIR Workshop on Information Retrieval and Applications of Graphical Models, Amsterdam. </li></ul> <ul><li> Sutskever, I. and Hinton, G. E. (2007) Learning multilevel distributed representations for high-dimensional sequences. AI and Statistics, 2007, Puerto Rico. </li></ul> <ul><li> Taylor, G. W., Hinton, G. E. and Roweis, S. (2007) Modeling human motion using binary latent variables. Advances in Neural Information Processing Systems 19, MIT Press, Cambridge, MA </li></ul> <ul><li> Welling, M., Rosen-Zvi, M., and Hinton, G. E. (2005). Exponential family harmoniums with an application to information retrieval. Advances in Neural Information Processing Systems 17, pages 1481-1488. MIT Press, Cambridge, MA. </li></ul> <p><b>Internal references</b> </p> <ul><li> Jan A. Sanders (2006) <a href="/article/Averaging" title="Averaging">Averaging</a>. <a href="/article/Scholarpedia" title="Scholarpedia">Scholarpedia</a>, 1(11):1760. </li></ul> <ul><li> Geoffrey E. Hinton (2007) <a href="/article/Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a>. Scholarpedia, 2(5):1668. </li></ul> <p><br /> </p> <h2> <span class="mw-headline" id="See_also">See also</span></h2>   <div class="cp-footer"><table cellpadding="0" border="0"><tr><td>Sponsored by: <a href="/article/User:Ke_CHEN" title="User:Ke CHEN"><span>Prof. Ke CHEN</span>, <span>School of Computer Science, The University of Manchester, U.K.</span></a></td></tr><tr><td><a rel="nofollow" class="external text" href="http://www.scholarpedia.org/w/index.php?title=Deep_belief_networks&oldid=61111">Reviewed by</a>: <a href="/article/User:Anonymous" title="User:Anonymous"><span>Anonymous</span></a></td></tr><tr><td><a rel="nofollow" class="external text" href="http://www.scholarpedia.org/w/index.php?title=Deep_belief_networks&oldid=61111">Reviewed by</a>: <a href="/article/User:Max_Welling" title="User:Max Welling"><span>Dr. Max Welling</span>, <span>School of Information and Computer Science, University of California, Irvine, CA</span></a></td></tr><tr><td><a rel="nofollow" class="external text" href="http://www.scholarpedia.org/w/index.php?title=Deep_belief_networks&oldid=61111">Reviewed by</a>: <a href="/article/User:Yoshua_Bengio" title="User:Yoshua Bengio"><span>Dr. Yoshua Bengio</span>, <span>Professor, department of computer science and operations research, Université de Montréal, Canada</span></a></td></tr><tr><td>Accepted on: <a rel="nofollow" class="external text" href="http://www.scholarpedia.org/w/index.php?title=Deep_belief_networks&oldid=61111">2009-04-11 01:15:10 GMT</a></td></tr></table></div> </div>   <div class="printfooter"> Retrieved from "<a href="http://www.scholarpedia.org/w/index.php?title=Deep_belief_networks&oldid=91189">http://www.scholarpedia.org/w/index.php?title=Deep_belief_networks&oldid=91189</a>" </div>   <div id='catlinks' class='catlinks'><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/article/Special:Categories" title="Special:Categories">Category</a>: <ul><li><a href="/article/Category:Pattern_Recognition" title="Category:Pattern Recognition">Pattern Recognition</a></li></ul></div></div>  <div class="visualClear"></div>   </div>  </div>   <div id="mw-head" class="noprint">  <div id="p-personal" class=""> <h5>Personal tools</h5> <ul> <li id="pt-login"><a href="/w/index.php?title=Special:UserLogin&returnto=Deep+belief+networks" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o">Log in</a></li> </ul> </div>  <div id="left-navigation">  <div id="p-namespaces" class="vectorTabs"> <h5>Namespaces</h5> <ul> <li id="ca-nstab-main" class="selected"><span><a href="/article/Deep_belief_networks" title="View the content page [c]" accesskey="c">Page</a></span></li> <li id="ca-talk"><span><a href="/article/Talk:Deep_belief_networks" title="Discussion about the content page [t]" accesskey="t">Discussion</a></span></li> </ul> </div>   <div id="p-variants" class="vectorMenu emptyPortlet"> <h4> </h4> <h5><span>Variants</span><a href="#"></a></h5> <div class="menu"> <ul> </ul> </div> </div>  </div> <div id="right-navigation">  <div id="p-views" class="vectorTabs"> <h5>Views</h5> <ul> <li id="ca-view" class="selected"><span><a href="/article/Deep_belief_networks" >Read</a></span></li> <li id="ca-viewsource"><span><a href="/w/index.php?title=Deep_belief_networks&action=edit" title="This page is protected.
You can view its source [e]" accesskey="e">View source</a></span></li> <li id="ca-history" class="collapsible"><span><a href="/w/index.php?title=Deep_belief_networks&action=history" title="Past revisions of this page [h]" accesskey="h">View history</a></span></li> </ul> </div>   <div id="p-cactions" class="vectorMenu emptyPortlet"> <h5><span>Actions</span><a href="#"></a></h5> <div class="menu"> <ul> </ul> </div> </div>   <div id="p-search"> <h5><label for="searchInput">Search</label></h5> <form action="/w/index.php" id="searchform"> <div id="simpleSearch"> <input name="search" title="Search Scholarpedia [f]" accesskey="f" id="searchInput" /> <button name="button" title="Search the pages for this text" id="searchButton"><img src="/w/skins/vector/images/search-ltr.png?303" alt="Search" /></button> <input type='hidden' name="title" value="Special:Search"/> </div> </form> </div>  </div> </div>   <div id="mw-panel" class="noprint">  <div id="p-logo"><a style="background-image: url(/w/skins/vector/images/splogo.png);" href="/article/Main_Page" title="Visit the main page"></a></div>   <div class="portal" id='p-navigation'> <h5>Navigation</h5> <div class="body"> <ul> <li id="n-mainpage-description"><a href="/article/Main_Page" title="Visit the main page [z]" accesskey="z">Main page</a></li> <li id="n-About"><a href="/article/Scholarpedia:About">About</a></li> <li id="n-Propose-a-new-article"><a href="/article/Special:ProposeArticle">Propose a new article</a></li> <li id="n-Instructions-for-Authors"><a href="/article/Scholarpedia:Instructions_for_Authors">Instructions for Authors</a></li> <li id="n-randompage"><a href="/article/Special:Random" title="Load a random page [x]" accesskey="x">Random article</a></li> <li id="n-FAQs"><a href="/article/Help:Frequently_Asked_Questions">FAQs</a></li> <li id="n-Help"><a href="/article/Scholarpedia:Help">Help</a></li> </ul> </div> </div>   <div class="portal" id='p-Focal_areas'> <h5>Focal areas</h5> <div class="body"> <ul> <li id="n-Astrophysics"><a href="/article/Encyclopedia:Astrophysics">Astrophysics</a></li> <li id="n-Celestial-mechanics"><a href="/article/Encyclopedia:Celestial_Mechanics">Celestial mechanics</a></li> <li id="n-Computational-neuroscience"><a href="/article/Encyclopedia:Computational_neuroscience">Computational neuroscience</a></li> <li id="n-Computational-intelligence"><a href="/article/Encyclopedia:Computational_intelligence">Computational intelligence</a></li> <li id="n-Dynamical-systems"><a href="/article/Encyclopedia:Dynamical_systems">Dynamical systems</a></li> <li id="n-Physics"><a href="/article/Encyclopedia:Physics">Physics</a></li> <li id="n-Touch"><a href="/article/Encyclopedia:Touch">Touch</a></li> <li id="n-More-topics"><a href="/article/Scholarpedia:Topics">More topics</a></li> </ul> </div> </div>   <div class="portal" id='p-Activity'> <h5>Activity</h5> <div class="body"> <ul> <li id="n-Recently-published-articles"><a href="/article/Special:RecentlyPublished">Recently published articles</a></li> <li id="n-Recently-sponsored-articles"><a href="/article/Special:RecentlySponsored">Recently sponsored articles</a></li> <li id="n-recentchanges"><a href="/article/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r">Recent changes</a></li> <li id="n-All-articles"><a href="/article/Special:AllPages">All articles</a></li> <li id="n-List-all-Curators"><a href="/article/Special:ListCurators">List all Curators</a></li> <li id="n-List-all-users"><a href="/article/Special:ListUsers">List all users</a></li> <li id="n-Journal"><a href="/article/Special:Journal">Scholarpedia Journal</a></li> </ul> </div> </div>     <div class="portal" id='p-tb'> <h5>Tools</h5> <div class="body"> <ul> <li id="t-whatlinkshere"><a href="/article/Special:WhatLinksHere/Deep_belief_networks" title="A list of all wiki pages that link here [j]" accesskey="j">What links here</a></li> <li id="t-recentchangeslinked"><a href="/article/Special:RecentChangesLinked/Deep_belief_networks" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li> <li id="t-specialpages"><a href="/article/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q">Special pages</a></li> <li><a href="/w/index.php?title=Deep_belief_networks&printable=yes" rel="alternate">Printable version</a></li> <li id="t-permalink"><a href="/w/index.php?title=Deep_belief_networks&oldid=91189" title="Permanent link to this revision of the page">Permanent link</a></li> </ul> </div> </div>    </div>   <div id="footer"> <div id="footer-icons"> <ul class="social"> <li><a href="https://twitter.com/scholarpedia" target="_blank"><img src="/w/skins/vector/images/twitter.png?303" /></a></li> <li><a href="https://plus.google.com/112873162496270574424" target="_blank"><img src="https://ssl.gstatic.com/images/icons/gplus-16.png" /></a></li> <li><a href="http://www.facebook.com/Scholarpedia" target="_blank"><img src="/w/skins/vector/images/facebook.png?303" /></a></li> <li><a href="http://www.linkedin.com/groups/Scholarpedia-4647975/about" target="_blank"><img src="/w/skins/vector/images/linkedin.png?303" /></a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-poweredbyico"> <a href="http://www.mediawiki.org/"><img src="/w/skins/common/images/poweredby_mediawiki_88x31.png" alt="Powered by MediaWiki" width="88" height="31" /></a> <a href="http://www.mathjax.org/"><img src="/w/skins/common/images/MathJaxBadge.gif" alt="Powered by MathJax" width="88" height="31" /></a> <a href="http://creativecommons.org/licenses/by-nc-sa/3.0/deed.en_US"><img src="/w/skins/common/88x31.png" alt="Creative Commons License" width="88" height="31" /></a> </li> </ul> </div> <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last modified on 21 October 2011, at 04:07.</li> <li id="footer-info-viewcount">This page has been accessed 315,416 times.</li> <li id="footer-info-copyright"> <span xmlns:dct="http://purl.org/dc/terms/" property="dct:title">"Deep belief networks"</span> by <a xmlns:cc="http://creativecommons.org/ns#" href="http://www.scholarpedia.org/article/Deep_belief_networks" property="cc:attributionName" rel="cc:attributionURL"> Geoffrey E. Hinton </a> is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/3.0/deed.en_US"> Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License</a>. Permissions beyond the scope of this license are described in the <a xmlns:cc="http://creativecommons.org/ns#" href="http://www.scholarpedia.org/article/Scholarpedia:Terms_of_use" rel="cc:morePermissions">Terms of Use</a></li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="/article/Scholarpedia:Privacy_policy" title="Scholarpedia:Privacy policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/article/Scholarpedia:About" class="mw-redirect" title="Scholarpedia:About">About Scholarpedia</a></li> <li id="footer-places-disclaimer"><a href="/article/Scholarpedia:General_disclaimer" title="Scholarpedia:General disclaimer">Disclaimers</a></li> </ul> <div style="clear:both"></div> </div>  <script src="http://www.scholarpedia.org/w/load.php?debug=false&lang=en&modules=skins.vector&only=scripts&skin=vector&*"></script> <script>if(window.mw){ mw.loader.load(["jquery.ui.dialog","curatorpedia.dashboard","curatorpedia.confirm","mediawiki.user","mediawiki.page.ready","ext.vector.collapsibleNav","ext.vector.collapsibleTabs","ext.vector.simpleSearch"], null, true); }</script> <script> var wgSitename = 'http://www.scholarpedia.org';</script> <script type='text/x-mathjax-config'> //<![CDATA[ MathJax.Hub.Config({ styles: { ".MathJax_Display": { display: "table-cell ! important", padding: "1em 0 ! important", width: (MathJax.Hub.Browser.isMSIE && (document.documentMode||0) < 8 ? "100% ! important" : "1000em ! important") } }, extensions: ["tex2jax.js","TeX/noErrors.js", "TeX/AMSmath.js","TeX/AMSsymbols.js"], jax: ["input/TeX", "output/HTML-CSS"], tex2jax: { inlineMath: [ ['$','$'], ["\$","\$"] ], displayMath: [ ['$$','$$'], ["\\[","\\]"] ], processEscapes: false, element: "content", ignoreClass: "(tex2jax_ignore|mw-search-results|searchresults)", /* note: this is part of a regex, check the docs! */ skipTags: ["script","noscript","style","textarea","code"] /* removed pre as wikimedia renders math in there */ }, TeX: { Macros: { /* Wikipedia compatibility: these macros are used on Wikipedia */ empty: '\\emptyset', P: '\\unicode{xb6}', Alpha: '\\unicode{x391}', /* FIXME: These capital Greeks don't show up in bold in \boldsymbol ... */ Beta: '\\unicode{x392}', Epsilon: '\\unicode{x395}', Zeta: '\\unicode{x396}', Eta: '\\unicode{x397}', Iota: '\\unicode{x399}', Kappa: '\\unicode{x39a}', Mu: '\\unicode{x39c}', Nu: '\\unicode{x39d}', Pi: '\\unicode{x3a0}', Rho: '\\unicode{x3a1}', Sigma: '\\unicode{x3a3}', Tau: '\\unicode{x3a4}', Chi: '\\unicode{x3a7}', C: '\\mathbb{C}', /* the complex numbers */ N: '\\mathbb{N}', /* the natural numbers */ Q: '\\mathbb{Q}', /* the rational numbers */ R: '\\mathbb{R}', /* the real numbers */ Z: '\\mathbb{Z}', /* the integer numbers */ /* some extre macros for ease of use; these are non-standard! */ F: '\\mathbb{F}', /* a finite field */ HH: '\\mathcal{H}', /* a Hilbert space */ bszero: '\\boldsymbol{0}', /* vector of zeros */ bsone: '\\boldsymbol{1}', /* vector of ones */ bst: '\\boldsymbol{t}', /* a vector 't' */ bsv: '\\boldsymbol{v}', /* a vector 'v' */ bsw: '\\boldsymbol{w}', /* a vector 'w' */ bsx: '\\boldsymbol{x}', /* a vector 'x' */ bsy: '\\boldsymbol{y}', /* a vector 'y' */ bsz: '\\boldsymbol{z}', /* a vector 'z' */ bsDelta: '\\boldsymbol{\\Delta}', /* a vector '\Delta' */ E: '\\mathrm{e}', /* the exponential */ rd: '\\,\\mathrm{d}', /* roman d for use in integrals: $\int f(x) \rd x$ */ rdelta: '\\,\\delta', /* delta operator for use in sums */ rD: '\\mathrm{D}', /* differential operator D */ /* example from MathJax on how to define macros with parameters: */ /* bold: ['{\\bf #1}', 1] */ RR: '\\mathbb{R}', ZZ: '\\mathbb{Z}', NN: '\\mathbb{N}', QQ: '\\mathbb{Q}', CC: '\\mathbb{C}', FF: '\\mathbb{F}' } } }); //]]> //<![CDATA[ MathJax.Hub.config.tex2jax.inlineMath.push(['$','$']); MathJax.Hub.config.tex2jax.displayMath.push(['$$','$$']); //]]> </script> <script type='text/javascript' src='https://cdn.mathjax.org/mathjax/2.3-latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML'></script> <script src="http://www.scholarpedia.org/w/load.php?debug=false&lang=en&modules=site&only=scripts&skin=vector&*"></script> <script type="text/javascript"> var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); </script> <script type="text/javascript"> var pageTracker = _gat._getTracker("UA-22078568-1"); pageTracker._initData(); pageTracker._trackPageview(); </script> </body> </html>