CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Recurrent neural network - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"40acfed7-02bf-47d6-b977-fdca7481d256","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Recurrent_neural_network","wgTitle":"Recurrent neural network","wgCurRevisionId":1257922012,"wgRevisionId":1257922012,"wgArticleId":1706303,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["CS1: long volume value","All articles with dead external links","Articles with dead external links from June 2024","Articles with permanently dead external links","CS1 Finnish-language sources (fi)","Articles with short description","Short description is different from Wikidata","All articles with unsourced statements","Articles with unsourced statements from June 2017","Neural network architectures"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en", "wgPageContentModel":"wikitext","wgRelevantPageName":"Recurrent_neural_network","wgRelevantArticleId":1706303,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":90000,"wgRelatedArticlesCompat":[],"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q1457734","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness", "fullVersionList","mobile","model","platform","platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.cite.styles":"ready","ext.math.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","mediawiki.page.media","ext.scribunto.logs","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP", "ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession","wikibase.sidebar.tracking"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.5"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Recurrent neural network - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Recurrent_neural_network"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Recurrent_neural_network&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Recurrent_neural_network"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Recurrent_neural_network rootpage-Recurrent_neural_network skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Main menu </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z">Main page</a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia">Contents</a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events">Current events</a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x">Random article</a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works">About Wikipedia</a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia">Contact us</a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia">Help</a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia">Learn to edit</a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors">Community portal</a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r">Recent changes</a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia">Upload file</a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"> Search </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Appearance </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en" class="">Donate</a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Recurrent+neural+network" title="You are encouraged to create an account and log in; however, it is not mandatory" class="">Create account</a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Recurrent+neural+network" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class="">Log in</a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Personal tools </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en">Donate</a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Recurrent+neural+network" title="You are encouraged to create an account and log in; however, it is not mandatory"> Create account</a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Recurrent+neural+network" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"> Log in</a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing">learn more</a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y">Contributions</a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n">Talk</a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-History" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#History"> <div class="vector-toc-text"> 1 History </div> </a> <button aria-controls="toc-History-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle History subsection </button> <ul id="toc-History-sublist" class="vector-toc-list"> <li id="toc-Before_modern" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Before_modern"> <div class="vector-toc-text"> 1.1 Before modern </div> </a> <ul id="toc-Before_modern-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Modern" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Modern"> <div class="vector-toc-text"> 1.2 Modern </div> </a> <ul id="toc-Modern-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Configurations" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Configurations"> <div class="vector-toc-text"> 2 Configurations </div> </a> <button aria-controls="toc-Configurations-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Configurations subsection </button> <ul id="toc-Configurations-sublist" class="vector-toc-list"> <li id="toc-Standard" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Standard"> <div class="vector-toc-text"> 2.1 Standard </div> </a> <ul id="toc-Standard-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Stacked_RNN" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Stacked_RNN"> <div class="vector-toc-text"> 2.2 Stacked RNN </div> </a> <ul id="toc-Stacked_RNN-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Bidirectional" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Bidirectional"> <div class="vector-toc-text"> 2.3 Bidirectional </div> </a> <ul id="toc-Bidirectional-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Encoder-decoder" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Encoder-decoder"> <div class="vector-toc-text"> 2.4 Encoder-decoder </div> </a> <ul id="toc-Encoder-decoder-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-PixelRNN" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#PixelRNN"> <div class="vector-toc-text"> 2.5 PixelRNN </div> </a> <ul id="toc-PixelRNN-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Architectures" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Architectures"> <div class="vector-toc-text"> 3 Architectures </div> </a> <button aria-controls="toc-Architectures-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Architectures subsection </button> <ul id="toc-Architectures-sublist" class="vector-toc-list"> <li id="toc-Fully_recurrent" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Fully_recurrent"> <div class="vector-toc-text"> 3.1 Fully recurrent </div> </a> <ul id="toc-Fully_recurrent-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Hopfield" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Hopfield"> <div class="vector-toc-text"> 3.2 Hopfield </div> </a> <ul id="toc-Hopfield-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Elman_networks_and_Jordan_networks" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Elman_networks_and_Jordan_networks"> <div class="vector-toc-text"> 3.3 Elman networks and Jordan networks </div> </a> <ul id="toc-Elman_networks_and_Jordan_networks-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Long_short-term_memory" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Long_short-term_memory"> <div class="vector-toc-text"> 3.4 Long short-term memory </div> </a> <ul id="toc-Long_short-term_memory-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Gated_recurrent_unit" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Gated_recurrent_unit"> <div class="vector-toc-text"> 3.5 Gated recurrent unit </div> </a> <ul id="toc-Gated_recurrent_unit-sublist" class="vector-toc-list"> <li id="toc-Bidirectional_associative_memory" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Bidirectional_associative_memory"> <div class="vector-toc-text"> 3.5.1 Bidirectional associative memory </div> </a> <ul id="toc-Bidirectional_associative_memory-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Echo_state" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Echo_state"> <div class="vector-toc-text"> 3.6 Echo state </div> </a> <ul id="toc-Echo_state-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Recursive" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Recursive"> <div class="vector-toc-text"> 3.7 Recursive </div> </a> <ul id="toc-Recursive-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Neural_Turing_machines" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Neural_Turing_machines"> <div class="vector-toc-text"> 3.8 Neural Turing machines </div> </a> <ul id="toc-Neural_Turing_machines-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Training" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Training"> <div class="vector-toc-text"> 4 Training </div> </a> <button aria-controls="toc-Training-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Training subsection </button> <ul id="toc-Training-sublist" class="vector-toc-list"> <li id="toc-Teacher_forcing" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Teacher_forcing"> <div class="vector-toc-text"> 4.1 Teacher forcing </div> </a> <ul id="toc-Teacher_forcing-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Gradient_descent" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Gradient_descent"> <div class="vector-toc-text"> 4.2 Gradient descent </div> </a> <ul id="toc-Gradient_descent-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Connectionist_temporal_classification" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Connectionist_temporal_classification"> <div class="vector-toc-text"> 4.3 Connectionist temporal classification </div> </a> <ul id="toc-Connectionist_temporal_classification-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Global_optimization_methods" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Global_optimization_methods"> <div class="vector-toc-text"> 4.4 Global optimization methods </div> </a> <ul id="toc-Global_optimization_methods-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Other_architectures" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Other_architectures"> <div class="vector-toc-text"> 5 Other architectures </div> </a> <button aria-controls="toc-Other_architectures-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Other architectures subsection </button> <ul id="toc-Other_architectures-sublist" class="vector-toc-list"> <li id="toc-Independently_RNN_(IndRNN)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Independently_RNN_(IndRNN)"> <div class="vector-toc-text"> 5.1 Independently RNN (IndRNN) </div> </a> <ul id="toc-Independently_RNN_(IndRNN)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Neural_history_compressor" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Neural_history_compressor"> <div class="vector-toc-text"> 5.2 Neural history compressor </div> </a> <ul id="toc-Neural_history_compressor-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Second_order_RNNs" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Second_order_RNNs"> <div class="vector-toc-text"> 5.3 Second order RNNs </div> </a> <ul id="toc-Second_order_RNNs-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Hierarchical_recurrent_neural_network" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Hierarchical_recurrent_neural_network"> <div class="vector-toc-text"> 5.4 Hierarchical recurrent neural network </div> </a> <ul id="toc-Hierarchical_recurrent_neural_network-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Recurrent_multilayer_perceptron_network" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Recurrent_multilayer_perceptron_network"> <div class="vector-toc-text"> 5.5 Recurrent multilayer perceptron network </div> </a> <ul id="toc-Recurrent_multilayer_perceptron_network-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Multiple_timescales_model" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Multiple_timescales_model"> <div class="vector-toc-text"> 5.6 Multiple timescales model </div> </a> <ul id="toc-Multiple_timescales_model-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Memristive_networks" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Memristive_networks"> <div class="vector-toc-text"> 5.7 Memristive networks </div> </a> <ul id="toc-Memristive_networks-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Continuous-time" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Continuous-time"> <div class="vector-toc-text"> 5.8 Continuous-time </div> </a> <ul id="toc-Continuous-time-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Libraries" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Libraries"> <div class="vector-toc-text"> 6 Libraries </div> </a> <ul id="toc-Libraries-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Applications" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Applications"> <div class="vector-toc-text"> 7 Applications </div> </a> <ul id="toc-Applications-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> 8 References </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Further_reading" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Further_reading"> <div class="vector-toc-text"> 9 Further reading </div> </a> <ul id="toc-Further_reading-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Toggle the table of contents </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading">Recurrent neural network</h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 28 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-28" aria-hidden="true" > 28 languages </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D8%A7%D9%84%D8%B4%D8%A8%D9%83%D8%A7%D8%AA_%D8%A7%D9%84%D8%B9%D8%B5%D8%A8%D9%8A%D8%A9_%D8%A7%D9%84%D9%85%D8%AA%D9%83%D8%B1%D8%B1%D8%A9" title="الشبكات العصبية المتكررة – Arabic" lang="ar" hreflang="ar" data-title="الشبكات العصبية المتكررة" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target">العربية</a></li><li class="interlanguage-link interwiki-bn mw-list-item"><a href="https://bn.wikipedia.org/wiki/%E0%A6%AA%E0%A7%81%E0%A6%A8%E0%A6%B0%E0%A6%BE%E0%A6%AC%E0%A7%83%E0%A6%A4%E0%A7%8D%E0%A6%A4_%E0%A6%B8%E0%A7%8D%E0%A6%A8%E0%A6%BE%E0%A6%AF%E0%A6%BC%E0%A7%81_%E0%A6%A8%E0%A7%87%E0%A6%9F%E0%A6%93%E0%A6%AF%E0%A6%BC%E0%A6%BE%E0%A6%B0%E0%A7%8D%E0%A6%95" title="পুনরাবৃত্ত স্নায়ু নেটওয়ার্ক – Bangla" lang="bn" hreflang="bn" data-title="পুনরাবৃত্ত স্নায়ু নেটওয়ার্ক" data-language-autonym="বাংলা" data-language-local-name="Bangla" class="interlanguage-link-target">বাংলা</a></li><li class="interlanguage-link interwiki-ca mw-list-item"><a href="https://ca.wikipedia.org/wiki/Xarxa_neuronal_recurrent" title="Xarxa neuronal recurrent – Catalan" lang="ca" hreflang="ca" data-title="Xarxa neuronal recurrent" data-language-autonym="Català" data-language-local-name="Catalan" class="interlanguage-link-target">Català</a></li><li class="interlanguage-link interwiki-cs mw-list-item"><a href="https://cs.wikipedia.org/wiki/Rekurentn%C3%AD_neuronov%C3%A1_s%C3%AD%C5%A5" title="Rekurentní neuronová síť – Czech" lang="cs" hreflang="cs" data-title="Rekurentní neuronová síť" data-language-autonym="Čeština" data-language-local-name="Czech" class="interlanguage-link-target">Čeština</a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Rekurrentes_neuronales_Netz" title="Rekurrentes neuronales Netz – German" lang="de" hreflang="de" data-title="Rekurrentes neuronales Netz" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target">Deutsch</a></li><li class="interlanguage-link interwiki-et mw-list-item"><a href="https://et.wikipedia.org/wiki/Rekurrentne_n%C3%A4rviv%C3%B5rk" title="Rekurrentne närvivõrk – Estonian" lang="et" hreflang="et" data-title="Rekurrentne närvivõrk" data-language-autonym="Eesti" data-language-local-name="Estonian" class="interlanguage-link-target">Eesti</a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Red_neuronal_recurrente" title="Red neuronal recurrente – Spanish" lang="es" hreflang="es" data-title="Red neuronal recurrente" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target">Español</a></li><li class="interlanguage-link interwiki-eu mw-list-item"><a href="https://eu.wikipedia.org/wiki/Neurona-sare_errepikakor" title="Neurona-sare errepikakor – Basque" lang="eu" hreflang="eu" data-title="Neurona-sare errepikakor" data-language-autonym="Euskara" data-language-local-name="Basque" class="interlanguage-link-target">Euskara</a></li><li class="interlanguage-link interwiki-fa mw-list-item"><a href="https://fa.wikipedia.org/wiki/%D8%B4%D8%A8%DA%A9%D9%87_%D8%B9%D8%B5%D8%A8%DB%8C_%D8%A8%D8%A7%D8%B2%DA%AF%D8%B4%D8%AA%DB%8C" title="شبکه عصبی بازگشتی – Persian" lang="fa" hreflang="fa" data-title="شبکه عصبی بازگشتی" data-language-autonym="فارسی" data-language-local-name="Persian" class="interlanguage-link-target">فارسی</a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/R%C3%A9seau_de_neurones_r%C3%A9currents" title="Réseau de neurones récurrents – French" lang="fr" hreflang="fr" data-title="Réseau de neurones récurrents" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target">Français</a></li><li class="interlanguage-link interwiki-gl mw-list-item"><a href="https://gl.wikipedia.org/wiki/Rede_neural_recorrente" title="Rede neural recorrente – Galician" lang="gl" hreflang="gl" data-title="Rede neural recorrente" data-language-autonym="Galego" data-language-local-name="Galician" class="interlanguage-link-target">Galego</a></li><li class="interlanguage-link interwiki-ko mw-list-item"><a href="https://ko.wikipedia.org/wiki/%EC%88%9C%ED%99%98_%EC%8B%A0%EA%B2%BD%EB%A7%9D" title="순환 신경망 – Korean" lang="ko" hreflang="ko" data-title="순환 신경망" data-language-autonym="한국어" data-language-local-name="Korean" class="interlanguage-link-target">한국어</a></li><li class="interlanguage-link interwiki-it mw-list-item"><a href="https://it.wikipedia.org/wiki/Rete_neurale_ricorrente" title="Rete neurale ricorrente – Italian" lang="it" hreflang="it" data-title="Rete neurale ricorrente" data-language-autonym="Italiano" data-language-local-name="Italian" class="interlanguage-link-target">Italiano</a></li><li class="interlanguage-link interwiki-mk mw-list-item"><a href="https://mk.wikipedia.org/wiki/%D0%A0%D0%B5%D0%BA%D1%83%D1%80%D0%B5%D0%BD%D1%82%D0%BD%D0%B8_%D0%BD%D0%B5%D0%B2%D1%80%D0%BE%D0%BD%D1%81%D0%BA%D0%B8_%D0%BC%D1%80%D0%B5%D0%B6%D0%B8" title="Рекурентни невронски мрежи – Macedonian" lang="mk" hreflang="mk" data-title="Рекурентни невронски мрежи" data-language-autonym="Македонски" data-language-local-name="Macedonian" class="interlanguage-link-target">Македонски</a></li><li class="interlanguage-link interwiki-ja mw-list-item"><a href="https://ja.wikipedia.org/wiki/%E5%9B%9E%E5%B8%B0%E5%9E%8B%E3%83%8B%E3%83%A5%E3%83%BC%E3%83%A9%E3%83%AB%E3%83%8D%E3%83%83%E3%83%88%E3%83%AF%E3%83%BC%E3%82%AF" title="回帰型ニューラルネットワーク – Japanese" lang="ja" hreflang="ja" data-title="回帰型ニューラルネットワーク" data-language-autonym="日本語" data-language-local-name="Japanese" class="interlanguage-link-target">日本語</a></li><li class="interlanguage-link interwiki-pl mw-list-item"><a href="https://pl.wikipedia.org/wiki/Rekurencyjna_sie%C4%87_neuronowa" title="Rekurencyjna sieć neuronowa – Polish" lang="pl" hreflang="pl" data-title="Rekurencyjna sieć neuronowa" data-language-autonym="Polski" data-language-local-name="Polish" class="interlanguage-link-target">Polski</a></li><li class="interlanguage-link interwiki-qu mw-list-item"><a href="https://qu.wikipedia.org/wiki/Kuti_kutiq_ankucha_llika" title="Kuti kutiq ankucha llika – Quechua" lang="qu" hreflang="qu" data-title="Kuti kutiq ankucha llika" data-language-autonym="Runa Simi" data-language-local-name="Quechua" class="interlanguage-link-target">Runa Simi</a></li><li class="interlanguage-link interwiki-ru mw-list-item"><a href="https://ru.wikipedia.org/wiki/%D0%A0%D0%B5%D0%BA%D1%83%D1%80%D1%80%D0%B5%D0%BD%D1%82%D0%BD%D0%B0%D1%8F_%D0%BD%D0%B5%D0%B9%D1%80%D0%BE%D0%BD%D0%BD%D0%B0%D1%8F_%D1%81%D0%B5%D1%82%D1%8C" title="Рекуррентная нейронная сеть – Russian" lang="ru" hreflang="ru" data-title="Рекуррентная нейронная сеть" data-language-autonym="Русский" data-language-local-name="Russian" class="interlanguage-link-target">Русский</a></li><li class="interlanguage-link interwiki-sk mw-list-item"><a href="https://sk.wikipedia.org/wiki/Rekurentn%C3%A1_neur%C3%B3nov%C3%A1_sie%C5%A5" title="Rekurentná neurónová sieť – Slovak" lang="sk" hreflang="sk" data-title="Rekurentná neurónová sieť" data-language-autonym="Slovenčina" data-language-local-name="Slovak" class="interlanguage-link-target">Slovenčina</a></li><li class="interlanguage-link interwiki-sr mw-list-item"><a href="https://sr.wikipedia.org/wiki/Rekurentna_neuronska_mre%C5%BEa" title="Rekurentna neuronska mreža – Serbian" lang="sr" hreflang="sr" data-title="Rekurentna neuronska mreža" data-language-autonym="Српски / srpski" data-language-local-name="Serbian" class="interlanguage-link-target">Српски / srpski</a></li><li class="interlanguage-link interwiki-fi mw-list-item"><a href="https://fi.wikipedia.org/wiki/Takaisinkytketty_neuroverkko" title="Takaisinkytketty neuroverkko – Finnish" lang="fi" hreflang="fi" data-title="Takaisinkytketty neuroverkko" data-language-autonym="Suomi" data-language-local-name="Finnish" class="interlanguage-link-target">Suomi</a></li><li class="interlanguage-link interwiki-th mw-list-item"><a href="https://th.wikipedia.org/wiki/%E0%B9%82%E0%B8%84%E0%B8%A3%E0%B8%87%E0%B8%82%E0%B9%88%E0%B8%B2%E0%B8%A2%E0%B8%9B%E0%B8%A3%E0%B8%B0%E0%B8%AA%E0%B8%B2%E0%B8%97%E0%B9%81%E0%B8%9A%E0%B8%9A%E0%B9%80%E0%B8%A7%E0%B8%B5%E0%B8%A2%E0%B8%99%E0%B8%8B%E0%B9%89%E0%B8%B3" title="โครงข่ายประสาทแบบเวียนซ้ำ – Thai" lang="th" hreflang="th" data-title="โครงข่ายประสาทแบบเวียนซ้ำ" data-language-autonym="ไทย" data-language-local-name="Thai" class="interlanguage-link-target">ไทย</a></li><li class="interlanguage-link interwiki-tr mw-list-item"><a href="https://tr.wikipedia.org/wiki/Yinelemeli_sinir_a%C4%9F%C4%B1" title="Yinelemeli sinir ağı – Turkish" lang="tr" hreflang="tr" data-title="Yinelemeli sinir ağı" data-language-autonym="Türkçe" data-language-local-name="Turkish" class="interlanguage-link-target">Türkçe</a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%A0%D0%B5%D0%BA%D1%83%D1%80%D0%B5%D0%BD%D1%82%D0%BD%D0%B0_%D0%BD%D0%B5%D0%B9%D1%80%D0%BE%D0%BD%D0%BD%D0%B0_%D0%BC%D0%B5%D1%80%D0%B5%D0%B6%D0%B0" title="Рекурентна нейронна мережа – Ukrainian" lang="uk" hreflang="uk" data-title="Рекурентна нейронна мережа" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target">Українська</a></li><li class="interlanguage-link interwiki-vi mw-list-item"><a href="https://vi.wikipedia.org/wiki/M%E1%BA%A1ng_th%E1%BA%A7n_kinh_h%E1%BB%93i_quy" title="Mạng thần kinh hồi quy – Vietnamese" lang="vi" hreflang="vi" data-title="Mạng thần kinh hồi quy" data-language-autonym="Tiếng Việt" data-language-local-name="Vietnamese" class="interlanguage-link-target">Tiếng Việt</a></li><li class="interlanguage-link interwiki-wuu mw-list-item"><a href="https://wuu.wikipedia.org/wiki/%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C" title="循环神经网络 – Wu" lang="wuu" hreflang="wuu" data-title="循环神经网络" data-language-autonym="吴语" data-language-local-name="Wu" class="interlanguage-link-target">吴语</a></li><li class="interlanguage-link interwiki-zh-yue mw-list-item"><a href="https://zh-yue.wikipedia.org/wiki/%E9%81%9E%E8%BF%B4%E7%A5%9E%E7%B6%93%E7%B6%B2%E7%B5%A1" title="遞迴神經網絡 – Cantonese" lang="yue" hreflang="yue" data-title="遞迴神經網絡" data-language-autonym="粵語" data-language-local-name="Cantonese" class="interlanguage-link-target">粵語</a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C" title="循环神经网络 – Chinese" lang="zh" hreflang="zh" data-title="循环神经网络" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target">中文</a></li> </ul> <div class="after-portlet after-portlet-lang"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q1457734#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Recurrent_neural_network" title="View the content page [c]" accesskey="c">Article</a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Recurrent_neural_network" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t">Talk</a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" >English </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Recurrent_neural_network">Read</a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&action=edit" title="Edit this page [e]" accesskey="e">Edit</a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&action=history" title="Past revisions of this page [h]" accesskey="h">View history</a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" >Tools </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Recurrent_neural_network">Read</a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&action=edit" title="Edit this page [e]" accesskey="e">Edit</a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&action=history">View history</a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Recurrent_neural_network" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j">What links here</a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Recurrent_neural_network" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li><li id="t-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u">Upload file</a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q">Special pages</a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&oldid=1257922012" title="Permanent link to this revision of this page">Permanent link</a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&action=info" title="More information about this page">Page information</a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Recurrent_neural_network&id=1257922012&wpFormIdentifier=titleform" title="Information on how to cite this page">Cite this page</a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FRecurrent_neural_network">Get shortened URL</a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FRecurrent_neural_network">Download QR code</a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Recurrent_neural_network&action=show-download-screen" title="Download this page as a PDF file">Download as PDF</a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Recurrent_neural_network&printable=yes" title="Printable version of this page [p]" accesskey="p">Printable version</a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q1457734" title="Structured data on this page hosted by Wikidata [g]" accesskey="g">Wikidata item</a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Class of artificial neural network</div> <style data-mw-deduplicate="TemplateStyles:r1236090951">.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}</style><div role="note" class="hatnote navigation-not-searchable">Not to be confused with <a href="/wiki/Recursive_neural_network" title="Recursive neural network">recursive neural network</a>.</div> <style data-mw-deduplicate="TemplateStyles:r1244144826">.mw-parser-output .machine-learning-list-title{background-color:#ddddff}html.skin-theme-clientpref-night .mw-parser-output .machine-learning-list-title{background-color:#222}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .machine-learning-list-title{background-color:#222}}</style> <style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1246091330">.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}</style><style data-mw-deduplicate="TemplateStyles:r886047488">.mw-parser-output .nobold{font-weight:normal}</style><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r886047488"><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="/wiki/Machine_learning" title="Machine learning">Machine learning</a> and <a href="/wiki/Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Paradigms</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Supervised_learning" title="Supervised learning">Supervised learning</a></li> <li><a href="/wiki/Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li> <li><a href="/wiki/Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li> <li><a href="/wiki/Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li> <li><a href="/wiki/Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li> <li><a href="/wiki/Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li> <li><a href="/wiki/Online_machine_learning" title="Online machine learning">Online learning</a></li> <li><a href="/wiki/Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li> <li><a href="/wiki/Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li> <li><a href="/wiki/Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li> <li><a href="/wiki/Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li> <li><a href="/wiki/Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li> <li><a href="/wiki/Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Problems</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Statistical_classification" title="Statistical classification">Classification</a></li> <li><a href="/wiki/Generative_model" title="Generative model">Generative modeling</a></li> <li><a href="/wiki/Regression_analysis" title="Regression analysis">Regression</a></li> <li><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></li> <li><a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li> <li><a href="/wiki/Density_estimation" title="Density estimation">Density estimation</a></li> <li><a href="/wiki/Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li> <li><a href="/wiki/Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li> <li><a href="/wiki/Automated_machine_learning" title="Automated machine learning">AutoML</a></li> <li><a href="/wiki/Association_rule_learning" title="Association rule learning">Association rules</a></li> <li><a href="/wiki/Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li> <li><a href="/wiki/Structured_prediction" title="Structured prediction">Structured prediction</a></li> <li><a href="/wiki/Feature_engineering" title="Feature engineering">Feature engineering</a></li> <li><a href="/wiki/Feature_learning" title="Feature learning">Feature learning</a></li> <li><a href="/wiki/Learning_to_rank" title="Learning to rank">Learning to rank</a></li> <li><a href="/wiki/Grammar_induction" title="Grammar induction">Grammar induction</a></li> <li><a href="/wiki/Ontology_learning" title="Ontology learning">Ontology learning</a></li> <li><a href="/wiki/Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="/wiki/Supervised_learning" title="Supervised learning">Supervised learning</a> (<a href="/wiki/Statistical_classification" title="Statistical classification">classification</a> • <a href="/wiki/Regression_analysis" title="Regression analysis">regression</a>) </div></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li> <li><a href="/wiki/Decision_tree_learning" title="Decision tree learning">Decision trees</a></li> <li><a href="/wiki/Ensemble_learning" title="Ensemble learning">Ensembles</a> <ul><li><a href="/wiki/Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li> <li><a href="/wiki/Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li> <li><a href="/wiki/Random_forest" title="Random forest">Random forest</a></li></ul></li> <li><a href="/wiki/K-nearest_neighbors_algorithm" title="K-nearest neighbors algorithm">k-NN</a></li> <li><a href="/wiki/Linear_regression" title="Linear regression">Linear regression</a></li> <li><a href="/wiki/Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li> <li><a href="/wiki/Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li> <li><a href="/wiki/Logistic_regression" title="Logistic regression">Logistic regression</a></li> <li><a href="/wiki/Perceptron" title="Perceptron">Perceptron</a></li> <li><a href="/wiki/Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li> <li><a href="/wiki/Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/BIRCH" title="BIRCH">BIRCH</a></li> <li><a href="/wiki/CURE_algorithm" title="CURE algorithm">CURE</a></li> <li><a href="/wiki/Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li> <li><a href="/wiki/K-means_clustering" title="K-means clustering">k-means</a></li> <li><a href="/wiki/Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li> <li><a href="/wiki/Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li> <li> <a href="/wiki/DBSCAN" title="DBSCAN">DBSCAN</a></li> <li><a href="/wiki/OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li> <li><a href="/wiki/Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Factor_analysis" title="Factor analysis">Factor analysis</a></li> <li><a href="/wiki/Canonical_correlation" title="Canonical correlation">CCA</a></li> <li><a href="/wiki/Independent_component_analysis" title="Independent component analysis">ICA</a></li> <li><a href="/wiki/Linear_discriminant_analysis" title="Linear discriminant analysis">LDA</a></li> <li><a href="/wiki/Non-negative_matrix_factorization" title="Non-negative matrix factorization">NMF</a></li> <li><a href="/wiki/Principal_component_analysis" title="Principal component analysis">PCA</a></li> <li><a href="/wiki/Proper_generalized_decomposition" title="Proper generalized decomposition">PGD</a></li> <li><a href="/wiki/T-distributed_stochastic_neighbor_embedding" title="T-distributed stochastic neighbor embedding">t-SNE</a></li> <li><a href="/wiki/Sparse_dictionary_learning" title="Sparse dictionary learning">SDL</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Graphical_model" title="Graphical model">Graphical models</a> <ul><li><a href="/wiki/Bayesian_network" title="Bayesian network">Bayes net</a></li> <li><a href="/wiki/Conditional_random_field" title="Conditional random field">Conditional random field</a></li> <li><a href="/wiki/Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Random_sample_consensus" title="Random sample consensus">RANSAC</a></li> <li><a href="/wiki/K-nearest_neighbors_algorithm" title="K-nearest neighbors algorithm">k-NN</a></li> <li><a href="/wiki/Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li> <li><a href="/wiki/Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural network</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Autoencoder" title="Autoencoder">Autoencoder</a></li> <li><a href="/wiki/Deep_learning" title="Deep learning">Deep learning</a></li> <li><a href="/wiki/Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li> <li><a class="mw-selflink selflink">Recurrent neural network</a> <ul><li><a href="/wiki/Long_short-term_memory" title="Long short-term memory">LSTM</a></li> <li><a href="/wiki/Gated_recurrent_unit" title="Gated recurrent unit">GRU</a></li> <li><a href="/wiki/Echo_state_network" title="Echo state network">ESN</a></li> <li><a href="/wiki/Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li> <li><a href="/wiki/Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a> <ul><li><a href="/wiki/Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li> <li><a href="/wiki/Generative_adversarial_network" title="Generative adversarial network">GAN</a></li> <li><a href="/wiki/Diffusion_model" title="Diffusion model">Diffusion model</a></li> <li><a href="/wiki/Self-organizing_map" title="Self-organizing map">SOM</a></li> <li><a href="/wiki/Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a> <ul><li><a href="/wiki/U-Net" title="U-Net">U-Net</a></li> <li><a href="/wiki/LeNet" title="LeNet">LeNet</a></li> <li><a href="/wiki/AlexNet" title="AlexNet">AlexNet</a></li> <li><a href="/wiki/DeepDream" title="DeepDream">DeepDream</a></li></ul></li> <li><a href="/wiki/Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li> <li><a href="/wiki/Transformer_(machine_learning_model)" class="mw-redirect" title="Transformer (machine learning model)">Transformer</a> <ul><li><a href="/wiki/Vision_transformer" title="Vision transformer">Vision</a></li></ul></li> <li><a href="/wiki/Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li> <li><a href="/wiki/Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li> <li><a href="/wiki/Memtransistor" title="Memtransistor">Memtransistor</a></li> <li><a href="/wiki/Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Q-learning" title="Q-learning">Q-learning</a></li> <li><a href="/wiki/State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li> <li><a href="/wiki/Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li> <li><a href="/wiki/Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a> <ul><li><a href="/wiki/Self-play_(reinforcement_learning_technique)" class="mw-redirect" title="Self-play (reinforcement learning technique)">Self-play</a></li></ul></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Learning with humans</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li> <li><a href="/wiki/Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li> <li><a href="/wiki/Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li> <li><a href="/wiki/Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">RLHF</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Model diagnostics</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li> <li><a href="/wiki/Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li> <li><a href="/wiki/Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li> <li><a href="/wiki/Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li> <li><a href="/wiki/Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li> <li><a href="/wiki/Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li> <li><a href="/wiki/Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li> <li><a href="/wiki/Occam_learning" title="Occam learning">Occam learning</a></li> <li><a href="/wiki/Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li> <li><a href="/wiki/Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li> <li><a href="/wiki/Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/ECML_PKDD" title="ECML PKDD">ECML PKDD</a></li> <li><a href="/wiki/Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li> <li><a href="/wiki/International_Conference_on_Machine_Learning" title="International Conference on Machine Learning">ICML</a></li> <li><a href="/wiki/International_Conference_on_Learning_Representations" title="International Conference on Learning Representations">ICLR</a></li> <li><a href="/wiki/International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li> <li><a href="/wiki/Machine_Learning_(journal)" title="Machine Learning (journal)">ML</a></li> <li><a href="/wiki/Journal_of_Machine_Learning_Research" title="Journal of Machine Learning Research">JMLR</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Related articles</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li> <li><a href="/wiki/List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a> <ul><li><a href="/wiki/List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li> <li><a href="/wiki/Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-navbar"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Machine_learning" title="Template:Machine learning"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Machine_learning" title="Template talk:Machine learning"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Machine_learning" title="Special:EditPage/Template:Machine learning"><abbr title="Edit this template">e</abbr></a></li></ul></div></td></tr></tbody></table> Recurrent neural networks (RNNs) are a class of <a href="/wiki/Neural_network_(machine_learning)" title="Neural network (machine learning)">artificial neural network</a> commonly used for sequential data processing. Unlike <a href="/wiki/Feedforward_neural_network" title="Feedforward neural network">feedforward neural networks</a>, which process data in a single pass, RNNs process data across multiple time steps, making them well-adapted for modelling and processing text, speech, and <a href="/wiki/Time_series" title="Time series">time series</a>.<a href="#cite_note-1">[1]</a> The building block of RNNs is the recurrent unit. This unit maintains a hidden state, essentially a form of memory, which is updated at each time step based on the current input and the previous hidden state. This feedback loop allows the network to learn from past inputs, and incorporate that knowledge into its current processing. Early RNNs suffered from the <a href="/wiki/Vanishing_gradient_problem" title="Vanishing gradient problem">vanishing gradient problem</a>, limiting their ability to learn long-range dependencies. This was solved by the <a href="/wiki/Long_short-term_memory" title="Long short-term memory">long short-term memory</a> (LSTM) variant in 1997, thus making it the standard architecture for RNN. RNNs have been applied to tasks such as unsegmented, connected <a href="/wiki/Handwriting_recognition" title="Handwriting recognition">handwriting recognition</a>,<a href="#cite_note-2">[2]</a> <a href="/wiki/Speech_recognition" title="Speech recognition">speech recognition</a>,<a href="#cite_note-sak2014-3">[3]</a><a href="#cite_note-liwu2015-4">[4]</a> <a href="/wiki/Natural_language_processing" title="Natural language processing">natural language processing</a>, and <a href="/wiki/Neural_machine_translation" title="Neural machine translation">neural machine translation</a>.<a href="#cite_note-5">[5]</a><a href="#cite_note-6">[6]</a> <style data-mw-deduplicate="TemplateStyles:r886046785">.mw-parser-output .toclimit-2 .toclevel-1 ul,.mw-parser-output .toclimit-3 .toclevel-2 ul,.mw-parser-output .toclimit-4 .toclevel-3 ul,.mw-parser-output .toclimit-5 .toclevel-4 ul,.mw-parser-output .toclimit-6 .toclevel-5 ul,.mw-parser-output .toclimit-7 .toclevel-6 ul{display:none}</style><div class="toclimit-3"><meta property="mw:PageProp/toc" /></div> <div class="mw-heading mw-heading2"><h2 id="History">History</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=1" title="Edit section: History">edit</a>]</div> <div class="mw-heading mw-heading3"><h3 id="Before_modern">Before modern</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=2" title="Edit section: Before modern">edit</a>]</div> One origin of RNN was neuroscience. The word "recurrent" is used to describe loop-like structures in anatomy. In 1901, <a href="/wiki/Santiago_Ram%C3%B3n_y_Cajal" title="Santiago Ramón y Cajal">Cajal</a> observed "recurrent semicircles" in the <a href="/wiki/Cerebellum" title="Cerebellum">cerebellar cortex</a> formed by <a href="/wiki/Parallel_fiber" class="mw-redirect" title="Parallel fiber">parallel fiber</a>, <a href="/wiki/Purkinje_cell" title="Purkinje cell">Purkinje cells</a>, and <a href="/wiki/Granule_cell" title="Granule cell">granule cells</a>.<a href="#cite_note-7">[7]</a><a href="#cite_note-8">[8]</a> In 1933, <a href="/wiki/Rafael_Lorente_de_N%C3%B3" title="Rafael Lorente de Nó">Lorente de Nó</a> discovered "recurrent, reciprocal connections" by <a href="/wiki/Golgi%27s_method" title="Golgi's method">Golgi's method</a>, and proposed that excitatory loops explain certain aspects of the <a href="/wiki/Vestibulo%E2%80%93ocular_reflex" title="Vestibulo–ocular reflex">vestibulo-ocular reflex</a>.<a href="#cite_note-9">[9]</a><a href="#cite_note-10">[10]</a> During 1940s, multiple people proposed the existence of feedback in the brain, which was a contrast to the previous understanding of the neural system as a purely feedforward structure. <a href="/wiki/Donald_O._Hebb" title="Donald O. Hebb">Hebb</a> considered "reverberating circuit" as an explanation for short-term memory.<a href="#cite_note-11">[11]</a> The McCulloch and Pitts paper (1943), which proposed the <a href="/wiki/McCulloch-Pitts_neuron" class="mw-redirect" title="McCulloch-Pitts neuron">McCulloch-Pitts neuron</a> model, considered networks that contains cycles. The current activity of such networks can be affected by activity indefinitely far in the past.<a href="#cite_note-12">[12]</a> They were both interested in closed loops as possible explanations for e.g. <a href="/wiki/Epilepsy" title="Epilepsy">epilepsy</a> and <a href="/wiki/Complex_regional_pain_syndrome" title="Complex regional pain syndrome">causalgia</a>.<a href="#cite_note-13">[13]</a><a href="#cite_note-14">[14]</a> <a href="/wiki/Renshaw_cell" title="Renshaw cell">Recurrent inhibition</a> was proposed in 1946 as a negative feedback mechanism in motor control. Neural feedback loops were a common topic of discussion at the <a href="/wiki/Macy_conferences" title="Macy conferences">Macy conferences</a>.<a href="#cite_note-15">[15]</a> See <a href="#cite_note-:0-16">[16]</a> for an extensive review of recurrent neural network models in neuroscience.<figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Typical_connections_in_a_close-loop_cross-coupled_perceptron.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/a/a5/Typical_connections_in_a_close-loop_cross-coupled_perceptron.png/220px-Typical_connections_in_a_close-loop_cross-coupled_perceptron.png" decoding="async" width="220" height="131" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/a/a5/Typical_connections_in_a_close-loop_cross-coupled_perceptron.png/330px-Typical_connections_in_a_close-loop_cross-coupled_perceptron.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/a/a5/Typical_connections_in_a_close-loop_cross-coupled_perceptron.png/440px-Typical_connections_in_a_close-loop_cross-coupled_perceptron.png 2x" data-file-width="503" data-file-height="300" /></a><figcaption>A close-loop cross-coupled perceptron network.<a href="#cite_note-:1-17">[17]</a>: 403, Fig. 47 .</figcaption></figure> <a href="/wiki/Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a> in 1960 published "close-loop cross-coupled perceptrons", which are 3-layered <a href="/wiki/Perceptron" title="Perceptron">perceptron</a> networks whose middle layer contains recurrent connections that change by a <a href="/wiki/Hebbian_theory" title="Hebbian theory">Hebbian learning</a> rule.<a href="#cite_note-18">[18]</a>: 73–75  Later, in Principles of Neurodynamics (1961), he described "closed-loop cross-coupled" and "back-coupled" perceptron networks, and made theoretical and experimental studies for Hebbian learning in these networks,<a href="#cite_note-:1-17">[17]</a>: Chapter 19, 21  and noted that a fully cross-coupled perceptron network is equivalent to an infinitely deep feedforward network.<a href="#cite_note-:1-17">[17]</a>: Section 19.11  Similar networks were published by Kaoru Nakano in 1971<a href="#cite_note-Nakano1971-19">[19]</a><a href="#cite_note-Nakano1972-20">[20]</a>,<a href="/wiki/Shun%27ichi_Amari" title="Shun'ichi Amari">Shun'ichi Amari</a> in 1972,<a href="#cite_note-Amari1972-21">[21]</a> and <a href="/w/index.php?title=William_A._Little_(physicist)&action=edit&redlink=1" class="new" title="William A. Little (physicist) (page does not exist)">William A. Little</a> [<a href="https://de.wikipedia.org/wiki/William_A._Little" class="extiw" title="de:William A. Little">de</a>] in 1974,<a href="#cite_note-little74-22">[22]</a> who was acknowledged by Hopfield in his 1982 paper. Another origin of RNN was <a href="/wiki/Statistical_mechanics" title="Statistical mechanics">statistical mechanics</a>. The <a href="/wiki/Ising_model" title="Ising model">Ising model</a> was developed by <a href="/wiki/Wilhelm_Lenz" title="Wilhelm Lenz">Wilhelm Lenz</a><a href="#cite_note-lenz1920-23">[23]</a> and <a href="/wiki/Ernst_Ising" title="Ernst Ising">Ernst Ising</a><a href="#cite_note-ising1925-24">[24]</a> in the 1920s<a href="#cite_note-25">[25]</a> as a simple statistical mechanical model of magnets at equilibrium. <a href="/wiki/Roy_J._Glauber" title="Roy J. Glauber">Glauber</a> in 1963 studied the Ising model evolving in time, as a process towards equilibrium (<a href="/wiki/Glauber_dynamics" title="Glauber dynamics">Glauber dynamics</a>), adding in the component of time.<a href="#cite_note-:22-26">[26]</a> The <a href="/wiki/Spin_glass" title="Spin glass">Sherrington–Kirkpatrick model</a> of spin glass, published in 1975,<a href="#cite_note-27">[27]</a> is the Hopfield network with random initialization. Sherrington and Kirkpatrick found that it is highly likely for the energy function of the SK model to have many local minima. In the 1982 paper, Hopfield applied this recently developed theory to study the Hopfield network with binary activation functions.<a href="#cite_note-Hopfield19822-28">[28]</a> In a 1984 paper he extended this to continuous activation functions.<a href="#cite_note-:02-29">[29]</a> It became a standard model for the study of neural networks through statistical mechanics.<a href="#cite_note-30">[30]</a><a href="#cite_note-31">[31]</a> <div class="mw-heading mw-heading3"><h3 id="Modern">Modern</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=3" title="Edit section: Modern">edit</a>]</div> Modern RNN networks are mainly based on two architectures: LSTM and BRNN.<a href="#cite_note-32">[32]</a> At the resurgence of neural networks in the 1980s, recurrent networks were studied again. They were sometimes called "iterated nets".<a href="#cite_note-33">[33]</a> Two early influential works were the <a href="#Jordan_network">Jordan network</a> (1986) and the <a href="#Elman_network">Elman network</a> (1990), which applied RNN to study <a href="/wiki/Cognitive_psychology" title="Cognitive psychology">cognitive psychology</a>. In 1993, a neural history compressor system solved a "Very Deep Learning" task that required more than 1000 subsequent <a href="/wiki/Layer_(deep_learning)" title="Layer (deep learning)">layers</a> in an RNN unfolded in time.<a href="#cite_note-schmidhuber1993-34">[34]</a> <a href="/wiki/Long_short-term_memory" title="Long short-term memory">Long short-term memory</a> (LSTM) networks were invented by <a href="/wiki/Sepp_Hochreiter" title="Sepp Hochreiter">Hochreiter</a> and <a href="/wiki/J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Schmidhuber</a> in 1995 and set accuracy records in multiple applications domains.<a href="#cite_note-35">[35]</a><a href="#cite_note-lstm-36">[36]</a> It became the default choice for RNN architecture. <a href="/wiki/Bidirectional_recurrent_neural_networks" title="Bidirectional recurrent neural networks">Bidirectional recurrent neural networks</a> (BRNN) uses two RNN that processes the same input in opposite directions.<a href="#cite_note-Schuster-37">[37]</a> These two are often combined, giving the bidirectional LSTM architecture. Around 2006, bidirectional LSTM started to revolutionize <a href="/wiki/Speech_recognition" title="Speech recognition">speech recognition</a>, outperforming traditional models in certain speech applications.<a href="#cite_note-38">[38]</a><a href="#cite_note-fernandez2007keyword-39">[39]</a> They also improved large-vocabulary speech recognition<a href="#cite_note-sak2014-3">[3]</a><a href="#cite_note-liwu2015-4">[4]</a> and <a href="/wiki/Text-to-speech" class="mw-redirect" title="Text-to-speech">text-to-speech</a> synthesis<a href="#cite_note-fan2015-40">[40]</a> and was used in <a href="/wiki/Google_Voice_Search" title="Google Voice Search">Google voice search</a>, and dictation on <a href="/wiki/Android_(operating_system)" title="Android (operating system)">Android devices</a>.<a href="#cite_note-sak2015-41">[41]</a> They broke records for improved <a href="/wiki/Machine_translation" title="Machine translation">machine translation</a>,<a href="#cite_note-sutskever2014-42">[42]</a> <a href="/wiki/Language_Modeling" class="mw-redirect" title="Language Modeling">language modeling</a><a href="#cite_note-vinyals2016-43">[43]</a> and Multilingual Language Processing.<a href="#cite_note-gillick2015-44">[44]</a> Also, LSTM combined with <a href="/wiki/Convolutional_neural_network" title="Convolutional neural network">convolutional neural networks</a> (CNNs) improved <a href="/wiki/Automatic_image_captioning" class="mw-redirect" title="Automatic image captioning">automatic image captioning</a>.<a href="#cite_note-vinyals2015-45">[45]</a> The idea of encoder-decoder sequence transduction had been developed in the early 2010s. The papers most commonly cited as the originators that produced seq2seq are two papers from 2014.<a href="#cite_note-:2-46">[46]</a><a href="#cite_note-sequence-47">[47]</a> A <a href="/wiki/Seq2seq" title="Seq2seq">seq2seq</a> architecture employs two RNN, typically LSTM, an "encoder" and a "decoder", for sequence transduction, such as machine translation. They became state of the art in machine translation, and was instrumental in the development of <a href="/wiki/Attention_(machine_learning)" title="Attention (machine learning)">attention mechanism</a> and <a href="/wiki/Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>. <div class="mw-heading mw-heading2"><h2 id="Configurations">Configurations</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=4" title="Edit section: Configurations">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Layer_(deep_learning)" title="Layer (deep learning)">Layer (deep learning)</a></div>An RNN-based model can be factored into two parts: configuration and architecture. Multiple RNN can be combined in a data flow, and the data flow itself is the configuration. Each RNN itself may have any architecture, including LSTM, GRU, etc. <div class="mw-heading mw-heading3"><h3 id="Standard">Standard</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=5" title="Edit section: Standard">edit</a>]</div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Recurrent_neural_network_unfold.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b5/Recurrent_neural_network_unfold.svg/220px-Recurrent_neural_network_unfold.svg.png" decoding="async" width="220" height="73" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b5/Recurrent_neural_network_unfold.svg/330px-Recurrent_neural_network_unfold.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b5/Recurrent_neural_network_unfold.svg/440px-Recurrent_neural_network_unfold.svg.png 2x" data-file-width="2126" data-file-height="709" /></a><figcaption>Compressed (left) and unfolded (right) basic recurrent neural network</figcaption></figure> RNNs come in many variants. Abstractly speaking, an RNN is a function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{\theta }}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{\theta }}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9874ae06066a2250709085e0fb521eebff2c2fb7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.143ex; height:2.509ex;" alt="{\displaystyle f_{\theta }}"> of type <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x_{t},h_{t})\mapsto (y_{t},h_{t+1})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">↦</mo> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (x_{t},h_{t})\mapsto (y_{t},h_{t+1})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/daede0f2ff9c38ad773254c9b68cdf2846a65de6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.852ex; height:2.843ex;" alt="{\displaystyle (x_{t},h_{t})\mapsto (y_{t},h_{t+1})}">, where <ul><li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f279a30bc8eabc788f3fe81c9cfb674e72e858db" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.156ex; height:2.009ex;" alt="{\displaystyle x_{t}}">: input vector;</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e8dbf3d8bfe322f68ff6400385578f8d78e1ba7c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.165ex; height:2.509ex;" alt="{\displaystyle h_{t}}">: hidden vector;</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0fe9554452b93508c9d2479414a45981ecc75a2d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.965ex; height:2.009ex;" alt="{\displaystyle y_{t}}">: output vector;</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>θ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \theta }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6e5ab2664b422d53eb0c7df3b87e1360d75ad9af" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.09ex; height:2.176ex;" alt="{\displaystyle \theta }">: neural network parameters.</li></ul> In words, it is a neural network that maps an input <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f279a30bc8eabc788f3fe81c9cfb674e72e858db" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.156ex; height:2.009ex;" alt="{\displaystyle x_{t}}"> into an output <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0fe9554452b93508c9d2479414a45981ecc75a2d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.965ex; height:2.009ex;" alt="{\displaystyle y_{t}}">, with the hidden vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e8dbf3d8bfe322f68ff6400385578f8d78e1ba7c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.165ex; height:2.509ex;" alt="{\displaystyle h_{t}}"> playing the role of "memory", a partial record of all previous input-output pairs. At each step, it transforms input to an output, and modifies its "memory" to help it to better perform future processing. The illustration to the right may be misleading to many because practical neural network topologies are frequently organized in "layers" and the drawing gives that appearance. However, what appears to be <a href="/wiki/Layer_(deep_learning)" title="Layer (deep learning)">layers</a> are, in fact, different steps in time, "unfolded" to produce the appearance of <a href="/wiki/Layer_(deep_learning)" title="Layer (deep learning)">layers</a>. <div class="mw-heading mw-heading3"><h3 id="Stacked_RNN">Stacked RNN</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=6" title="Edit section: Stacked RNN">edit</a>]</div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Stacked_RNN.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/e/e3/Stacked_RNN.png/220px-Stacked_RNN.png" decoding="async" width="220" height="108" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/e/e3/Stacked_RNN.png/330px-Stacked_RNN.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/e/e3/Stacked_RNN.png/440px-Stacked_RNN.png 2x" data-file-width="1426" data-file-height="700" /></a><figcaption>Stacked RNN.</figcaption></figure>A stacked RNN, or deep RNN, is composed of multiple RNNs stacked one above the other. Abstractly, it is structured as follows <ol><li>Layer 1 has hidden vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{1,t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{1,t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41d508e0dc0b303c58fa98ac785044f0d302024c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.444ex; height:2.843ex;" alt="{\displaystyle h_{1,t}}">, parameters <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta _{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \theta _{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7f84b9443d095623e02fd287cd095123d70b0278" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.145ex; height:2.509ex;" alt="{\displaystyle \theta _{1}}">, and maps <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{\theta _{1}}:(x_{0,t},h_{1,t})\mapsto (x_{1,t},h_{1,t+1})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </msub> <mo>:</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">↦</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mi>t</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{\theta _{1}}:(x_{0,t},h_{1,t})\mapsto (x_{1,t},h_{1,t+1})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3d6c58b510835cafedab19747e1a57a95bfbbc28" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:30.071ex; height:3.009ex;" alt="{\displaystyle f_{\theta _{1}}:(x_{0,t},h_{1,t})\mapsto (x_{1,t},h_{1,t+1})}">.</li> <li>Layer 2 has hidden vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{2,t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{2,t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/30a4a50b190be9e0f1df9770bb3d42879f564aff" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.444ex; height:2.843ex;" alt="{\displaystyle h_{2,t}}">, parameters <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta _{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \theta _{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0ed6ea624b20b153403979ffaf5434fc36de2990" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.145ex; height:2.509ex;" alt="{\displaystyle \theta _{2}}">, and maps <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{\theta _{2}}:(x_{1,t},h_{2,t})\mapsto (x_{2,t},h_{2,t+1})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> </msub> <mo>:</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">↦</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mi>t</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{\theta _{2}}:(x_{1,t},h_{2,t})\mapsto (x_{2,t},h_{2,t+1})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c936d144b72db51f32a1fc4af94e60849bfe2ffe" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:30.071ex; height:3.009ex;" alt="{\displaystyle f_{\theta _{2}}:(x_{1,t},h_{2,t})\mapsto (x_{2,t},h_{2,t+1})}">.</li> <li>...</li> <li>Layer <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>n</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a601995d55609f2d9f5e233e36fbe9ea26011b3b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}"> has hidden vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{n,t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>,</mo> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{n,t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bc5bbbf6e890a622b1a1a9b492ab4f0b1c01d843" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.609ex; height:2.843ex;" alt="{\displaystyle h_{n,t}}">, parameters <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta _{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \theta _{n}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/79cc00920259451fc1a684ba7350b6f93ce4f08a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.309ex; height:2.509ex;" alt="{\displaystyle \theta _{n}}">, and maps <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{\theta _{n}}:(x_{n-1,t},h_{n,t})\mapsto (x_{n,t},h_{n,t+1})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mrow> </msub> <mo>:</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">↦</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>,</mo> <mi>t</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>,</mo> <mi>t</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{\theta _{n}}:(x_{n-1,t},h_{n,t})\mapsto (x_{n,t},h_{n,t+1})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ea881abbb0a57bb90425e6fe8c2cee342f8bef1c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:32.962ex; height:3.009ex;" alt="{\displaystyle f_{\theta _{n}}:(x_{n-1,t},h_{n,t})\mapsto (x_{n,t},h_{n,t+1})}">.</li></ol> Each layer operates as a stand-alone RNN, and each layer's output sequence is used as the input sequence to the layer above. There is no conceptual limit to the depth of stacked RNN. <div class="mw-heading mw-heading3"><h3 id="Bidirectional">Bidirectional</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=7" title="Edit section: Bidirectional">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Bidirectional_recurrent_neural_networks" title="Bidirectional recurrent neural networks">Bidirectional recurrent neural networks</a></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Bidirectional_RNN.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/f/f0/Bidirectional_RNN.png/220px-Bidirectional_RNN.png" decoding="async" width="220" height="108" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/f/f0/Bidirectional_RNN.png/330px-Bidirectional_RNN.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/f/f0/Bidirectional_RNN.png/440px-Bidirectional_RNN.png 2x" data-file-width="1426" data-file-height="700" /></a><figcaption>Bidirectional RNN.</figcaption></figure>A bidirectional RNN (biRNN) is composed of two RNNs, one processing the input sequence in one direction, and another in the opposite direction. Abstractly, it is structured as follows: <ul><li>The forward RNN processes in one direction: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{\theta }(x_{0},h_{0})=(y_{0},h_{1}),f_{\theta }(x_{1},h_{1})=(y_{1},h_{2}),\dots }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>,</mo> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>,</mo> <mo>…</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{\theta }(x_{0},h_{0})=(y_{0},h_{1}),f_{\theta }(x_{1},h_{1})=(y_{1},h_{2}),\dots }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f6129c199ab8b86d643b7c35e4655a42e36c0ea6" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:45.374ex; height:2.843ex;" alt="{\displaystyle f_{\theta }(x_{0},h_{0})=(y_{0},h_{1}),f_{\theta }(x_{1},h_{1})=(y_{1},h_{2}),\dots }"></li> <li>The backward RNN processes in the opposite direction:<math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f'_{\theta '}(x_{N},h_{N}')=(y'_{N},h_{N-1}'),f'_{\theta '}(x_{N-1},h_{N-1}')=(y'_{N-1},h_{N-2}'),\dots }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msubsup> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <msup> <mi>θ</mi> <mo>′</mo> </msup> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </msub> <mo>,</mo> <msubsup> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo>=</mo> <mo stretchy="false">(</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> <mo>′</mo> </msubsup> <mo>,</mo> <msubsup> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo>−</mo> <mn>1</mn> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo>,</mo> <msubsup> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <msup> <mi>θ</mi> <mo>′</mo> </msup> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msubsup> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo>−</mo> <mn>1</mn> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo>=</mo> <mo stretchy="false">(</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo>−</mo> <mn>1</mn> </mrow> <mo>′</mo> </msubsup> <mo>,</mo> <msubsup> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo>−</mo> <mn>2</mn> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo>,</mo> <mo>…</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f'_{\theta '}(x_{N},h_{N}')=(y'_{N},h_{N-1}'),f'_{\theta '}(x_{N-1},h_{N-1}')=(y'_{N-1},h_{N-2}'),\dots }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2f5f33f98c860de2b99b57d62dd99c59ebc80fc0" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:62.037ex; height:3.343ex;" alt="{\displaystyle f'_{\theta '}(x_{N},h_{N}')=(y'_{N},h_{N-1}'),f'_{\theta '}(x_{N-1},h_{N-1}')=(y'_{N-1},h_{N-2}'),\dots }"></li></ul> The two output sequences are then concatenated to give the total output: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle ((y_{0},y_{0}'),(y_{1},y_{1}'),\dots ,(y_{N},y_{N}'))}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>,</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo>,</mo> <mo>…</mo> <mo>,</mo> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </msub> <mo>,</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> <mo>′</mo> </msubsup> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle ((y_{0},y_{0}'),(y_{1},y_{1}'),\dots ,(y_{N},y_{N}'))}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6b4f2aede172075e4a2d3ef5c525448fb5e241ae" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:30.986ex; height:3.009ex;" alt="{\displaystyle ((y_{0},y_{0}'),(y_{1},y_{1}'),\dots ,(y_{N},y_{N}'))}">. Bidirectional RNN allows the model to process a token both in the context of what came before it and what came after it. By stacking multiple bidirectional RNNs together, the model can process a token increasingly contextually. The <a href="/wiki/ELMo" title="ELMo">ELMo</a> model (2018)<a href="#cite_note-48">[48]</a> is a stacked bidirectional <a href="/wiki/Long_short-term_memory" title="Long short-term memory">LSTM</a> which takes character-level as inputs and produces word-level embeddings. <div class="mw-heading mw-heading3"><h3 id="Encoder-decoder">Encoder-decoder</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=8" title="Edit section: Encoder-decoder">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Seq2seq" title="Seq2seq">seq2seq</a></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Decoder_RNN.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/7/7c/Decoder_RNN.png/220px-Decoder_RNN.png" decoding="async" width="220" height="95" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/7/7c/Decoder_RNN.png/330px-Decoder_RNN.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/7/7c/Decoder_RNN.png/440px-Decoder_RNN.png 2x" data-file-width="1426" data-file-height="613" /></a><figcaption>A decoder without an encoder.</figcaption></figure> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Seq2seq_RNN_encoder-decoder_with_attention_mechanism,_training_and_inferring.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/7/72/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png/220px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png" decoding="async" width="220" height="97" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/7/72/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png/330px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/7/72/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png/440px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png 2x" data-file-width="1426" data-file-height="630" /></a><figcaption>Encoder-decoder RNN without attention mechanism.</figcaption></figure> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Seq2seq_RNN_encoder-decoder_with_attention_mechanism,_training.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/c/c7/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training.png/220px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training.png" decoding="async" width="220" height="120" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/c/c7/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training.png/330px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/c/c7/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training.png/440px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training.png 2x" data-file-width="1426" data-file-height="775" /></a><figcaption>Encoder-decoder RNN with attention mechanism.</figcaption></figure> Two RNNs can be run front-to-back in an encoder-decoder configuration. The encoder RNN processes an input sequence into a sequence of hidden vectors, and the decoder RNN processes the sequence of hidden vectors to an output sequence, with an optional <a href="/wiki/Attention_(machine_learning)" title="Attention (machine learning)">attention mechanism</a>. This was used to construct state of the art <a href="/wiki/Neural_machine_translation" title="Neural machine translation">neural machine translators</a> during the 2014–2017 period. This was an instrumental step towards the development of <a href="/wiki/Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformers</a>.<a href="#cite_note-49">[49]</a> <div class="mw-heading mw-heading3"><h3 id="PixelRNN">PixelRNN</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=9" title="Edit section: PixelRNN">edit</a>]</div> An RNN may process data with more than one dimension. PixelRNN processes two-dimensional data, with many possible directions.<a href="#cite_note-50">[50]</a> For example, the row-by-row direction processes an <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n\times n}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>n</mi> <mo>×</mo> <mi>n</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n\times n}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/59d2b4cb72e304526cf5b5887147729ea259da78" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.63ex; height:1.676ex;" alt="{\displaystyle n\times n}"> grid of vectors <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{i,j}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>,</mo> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{i,j}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fd698068c9322e82d11cb2bc02cb2f51739a2cc0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.264ex; height:2.343ex;" alt="{\displaystyle x_{i,j}}"> in the following order: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{1,1},x_{1,2},\dots ,x_{1,n},x_{2,1},x_{2,2},\dots ,x_{2,n},\dots ,x_{n,n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>,</mo> <mi>n</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> <mo>,</mo> <mi>n</mi> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>,</mo> <mi>n</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{1,1},x_{1,2},\dots ,x_{1,n},x_{2,1},x_{2,2},\dots ,x_{2,n},\dots ,x_{n,n}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6b9de0f040f133e51ec188e98ed9541a98705908" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:44.936ex; height:2.343ex;" alt="{\displaystyle x_{1,1},x_{1,2},\dots ,x_{1,n},x_{2,1},x_{2,2},\dots ,x_{2,n},\dots ,x_{n,n}}">The diagonal BiLSTM uses two LSTMs to process the same grid. One processes it from the top-left corner to the bottom-right, such that it processes <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{i,j}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>,</mo> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{i,j}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fd698068c9322e82d11cb2bc02cb2f51739a2cc0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.264ex; height:2.343ex;" alt="{\displaystyle x_{i,j}}"> depending on its hidden state and cell state on the top and the left side: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{i-1,j},c_{i-1,j}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>−</mo> <mn>1</mn> <mo>,</mo> <mi>j</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>c</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>−</mo> <mn>1</mn> <mo>,</mo> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{i-1,j},c_{i-1,j}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ec3abb324832ed758f9376976cae69702169a7df" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:11.45ex; height:2.843ex;" alt="{\displaystyle h_{i-1,j},c_{i-1,j}}"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{i,j-1},c_{i,j-1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>,</mo> <mi>j</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>c</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>,</mo> <mi>j</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{i,j-1},c_{i,j-1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/82483890bf0f5558df1cee92b40c57d6c0e29228" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:11.45ex; height:2.843ex;" alt="{\displaystyle h_{i,j-1},c_{i,j-1}}">. The other processes it from the top-right corner to the bottom-left. <div class="mw-heading mw-heading2"><h2 id="Architectures">Architectures</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=10" title="Edit section: Architectures">edit</a>]</div> <div class="mw-heading mw-heading3"><h3 id="Fully_recurrent">Fully recurrent</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=11" title="Edit section: Fully recurrent">edit</a>]</div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Hopfield-net-vector.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/4/44/Hopfield-net-vector.svg/220px-Hopfield-net-vector.svg.png" decoding="async" width="220" height="252" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/4/44/Hopfield-net-vector.svg/330px-Hopfield-net-vector.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/4/44/Hopfield-net-vector.svg/440px-Hopfield-net-vector.svg.png 2x" data-file-width="730" data-file-height="835" /></a><figcaption>A fully connected RNN with 4 neurons.</figcaption></figure> Fully recurrent neural networks (FRNN) connect the outputs of all neurons to the inputs of all neurons. In other words, it is a <a href="/wiki/Fully_connected_network" class="mw-redirect" title="Fully connected network">fully connected network</a>. This is the most general neural network topology, because all other topologies can be represented by setting some connection weights to zero to simulate the lack of connections between those neurons. <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:RNN_architecture.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/0/02/RNN_architecture.png/220px-RNN_architecture.png" decoding="async" width="220" height="81" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/0/02/RNN_architecture.png/330px-RNN_architecture.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/0/02/RNN_architecture.png/440px-RNN_architecture.png 2x" data-file-width="1426" data-file-height="524" /></a><figcaption>A simple Elman network where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{h}=\tanh ,\sigma _{y}={\text{Identity}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <mo>=</mo> <mi>tanh</mi> <mo>,</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mtext>Identity</mtext> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \sigma _{h}=\tanh ,\sigma _{y}={\text{Identity}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/02121b05b248e1d48a45e75052bc4e5144c8bde7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:24.907ex; height:2.843ex;" alt="{\displaystyle \sigma _{h}=\tanh ,\sigma _{y}={\text{Identity}}}">.</figcaption></figure> <div class="mw-heading mw-heading3"><h3 id="Hopfield">Hopfield</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=12" title="Edit section: Hopfield">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Hopfield_network" title="Hopfield network">Hopfield network</a></div> The <a href="/wiki/Hopfield_network" title="Hopfield network">Hopfield network</a> is an RNN in which all connections across layers are equally sized. It requires <a href="/wiki/Stationary_process" title="Stationary process">stationary</a> inputs and is thus not a general RNN, as it does not process sequences of patterns. However, it guarantees that it will converge. If the connections are trained using <a href="/wiki/Hebbian_learning" class="mw-redirect" title="Hebbian learning">Hebbian learning</a>, then the Hopfield network can perform as <a href="/wiki/Robustness_(computer_science)" title="Robustness (computer science)">robust</a> <a href="/wiki/Content-addressable_memory" title="Content-addressable memory">content-addressable memory</a>, resistant to connection alteration. <div class="mw-heading mw-heading3"><h3 id="Elman_networks_and_Jordan_networks">Elman networks and Jordan networks</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=13" title="Edit section: Elman networks and Jordan networks">edit</a>]</div> <figure class="mw-default-size mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:Elman_srnn.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/8/8f/Elman_srnn.png/220px-Elman_srnn.png" decoding="async" width="220" height="244" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/8f/Elman_srnn.png/330px-Elman_srnn.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/8f/Elman_srnn.png/440px-Elman_srnn.png 2x" data-file-width="673" data-file-height="745" /></a><figcaption>The Elman network</figcaption></figure> An <a href="/wiki/Jeff_Elman" class="mw-redirect" title="Jeff Elman">Elman</a> network is a three-layer network (arranged horizontally as x, y, and z in the illustration) with the addition of a set of context units (u in the illustration). The middle (hidden) layer is connected to these context units fixed with a weight of one.<a href="#cite_note-bmm615-51">[51]</a> At each time step, the input is fed forward and a <a href="/wiki/Learning_rule" title="Learning rule">learning rule</a> is applied. The fixed back-connections save a copy of the previous values of the hidden units in the context units (since they propagate over the connections before the learning rule is applied). Thus the network can maintain a sort of state, allowing it to perform tasks such as sequence-prediction that are beyond the power of a standard <a href="/wiki/Multilayer_perceptron" title="Multilayer perceptron">multilayer perceptron</a>. <a href="/wiki/Michael_I._Jordan" title="Michael I. Jordan">Jordan</a> networks are similar to Elman networks. The context units are fed from the output layer instead of the hidden layer. The context units in a Jordan network are also called the state layer. They have a recurrent connection to themselves.<a href="#cite_note-bmm615-51">[51]</a> Elman and Jordan networks are also known as "Simple recurrent networks" (SRN). <dl><dt>Elman network<a href="#cite_note-52">[52]</a></dt> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}h_{t}&=\sigma _{h}(W_{h}x_{t}+U_{h}h_{t-1}+b_{h})\\y_{t}&=\sigma _{y}(W_{y}h_{t}+b_{y})\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>W</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>+</mo> <msub> <mi>U</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>W</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>+</mo> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}h_{t}&=\sigma _{h}(W_{h}x_{t}+U_{h}h_{t-1}+b_{h})\\y_{t}&=\sigma _{y}(W_{y}h_{t}+b_{y})\end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/983c33bfddb6c6d8b1bfdf9accf50bb5634e2f5a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:30.748ex; height:6.176ex;" alt="{\displaystyle {\begin{aligned}h_{t}&=\sigma _{h}(W_{h}x_{t}+U_{h}h_{t-1}+b_{h})\\y_{t}&=\sigma _{y}(W_{y}h_{t}+b_{y})\end{aligned}}}"></dd> <dt>Jordan network<a href="#cite_note-53">[53]</a></dt> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}h_{t}&=\sigma _{h}(W_{h}x_{t}+U_{h}s_{t}+b_{h})\\y_{t}&=\sigma _{y}(W_{y}h_{t}+b_{y})\\s_{t}&=\sigma _{s}(W_{s,s}s_{t-1}+W_{s,y}y_{t-1}+b_{s})\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>W</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>+</mo> <msub> <mi>U</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <msub> <mi>s</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>+</mo> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>h</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>W</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> <mo>+</mo> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <msub> <mi>s</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>s</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>W</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>s</mi> <mo>,</mo> <mi>s</mi> </mrow> </msub> <msub> <mi>s</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>W</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>s</mi> <mo>,</mo> <mi>y</mi> </mrow> </msub> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>s</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}h_{t}&=\sigma _{h}(W_{h}x_{t}+U_{h}s_{t}+b_{h})\\y_{t}&=\sigma _{y}(W_{y}h_{t}+b_{y})\\s_{t}&=\sigma _{s}(W_{s,s}s_{t-1}+W_{s,y}y_{t-1}+b_{s})\end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/da448ff637dcc68dbd497e10f7623c76c265b983" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -4.171ex; width:34.816ex; height:9.509ex;" alt="{\displaystyle {\begin{aligned}h_{t}&=\sigma _{h}(W_{h}x_{t}+U_{h}s_{t}+b_{h})\\y_{t}&=\sigma _{y}(W_{y}h_{t}+b_{y})\\s_{t}&=\sigma _{s}(W_{s,s}s_{t-1}+W_{s,y}y_{t-1}+b_{s})\end{aligned}}}"></dd></dl> Variables and functions <ul><li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f279a30bc8eabc788f3fe81c9cfb674e72e858db" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.156ex; height:2.009ex;" alt="{\displaystyle x_{t}}">: input vector</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e8dbf3d8bfe322f68ff6400385578f8d78e1ba7c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.165ex; height:2.509ex;" alt="{\displaystyle h_{t}}">: hidden layer vector</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>s</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/92a402d151a0173378ee252a634c77898ebe4b06" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.916ex; height:2.009ex;" alt="{\displaystyle s_{t}}">: "state" vector,</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{t}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>t</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{t}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0fe9554452b93508c9d2479414a45981ecc75a2d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.965ex; height:2.009ex;" alt="{\displaystyle y_{t}}">: output vector</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>W</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle W}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/54a9c4c547f4d6111f81946cad242b18298d70b7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.176ex;" alt="{\displaystyle W}">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle U}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>U</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle U}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/458a728f53b9a0274f059cd695e067c430956025" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.783ex; height:2.176ex;" alt="{\displaystyle U}"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>b</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle b}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f11423fbb2e967f986e36804a8ae4271734917c3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.998ex; height:2.176ex;" alt="{\displaystyle b}">: parameter matrices and vector</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>σ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \sigma }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/59f59b7c3e6fdb1d0365a494b81fb9a696138c36" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle \sigma }">: <a href="/wiki/Activation_function" title="Activation function">Activation functions</a></li></ul> <div class="mw-heading mw-heading3"><h3 id="Long_short-term_memory">Long short-term memory</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=14" title="Edit section: Long short-term memory">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Long_short-term_memory" title="Long short-term memory">Long short-term memory</a></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Long_Short-Term_Memory.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/6/63/Long_Short-Term_Memory.svg/220px-Long_Short-Term_Memory.svg.png" decoding="async" width="220" height="98" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/6/63/Long_Short-Term_Memory.svg/330px-Long_Short-Term_Memory.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/6/63/Long_Short-Term_Memory.svg/440px-Long_Short-Term_Memory.svg.png 2x" data-file-width="1594" data-file-height="709" /></a><figcaption>Long short-term memory unit</figcaption></figure> Long short-term memory (LSTM) is the most widely used RNN architecture. It was designed to solve the <a href="/wiki/Vanishing_gradient_problem" title="Vanishing gradient problem">vanishing gradient problem</a>. LSTM is normally augmented by recurrent gates called "forget gates".<a href="#cite_note-gers2002-54">[54]</a> LSTM prevents backpropagated errors from vanishing or exploding.<a href="#cite_note-hochreiter1991-55">[55]</a> Instead, errors can flow backward through unlimited numbers of virtual layers unfolded in space. That is, LSTM can learn tasks that require memories of events that happened thousands or even millions of discrete time steps earlier. Problem-specific LSTM-like topologies can be evolved.<a href="#cite_note-bayer2009-56">[56]</a> LSTM works even given long delays between significant events and can handle signals that mix low and high-frequency components. Many applications use stacks of LSTMs,<a href="#cite_note-fernandez2007-57">[57]</a> for which it is called "deep LSTM". LSTM can learn to recognize <a href="/wiki/Context-sensitive_languages" class="mw-redirect" title="Context-sensitive languages">context-sensitive languages</a> unlike previous models based on <a href="/wiki/Hidden_Markov_model" title="Hidden Markov model">hidden Markov models</a> (HMM) and similar concepts.<a href="#cite_note-peepholeLSTM-58">[58]</a> <div class="mw-heading mw-heading3"><h3 id="Gated_recurrent_unit">Gated recurrent unit</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=15" title="Edit section: Gated recurrent unit">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit</a></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Gated_Recurrent_Unit.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/5/5f/Gated_Recurrent_Unit.svg/220px-Gated_Recurrent_Unit.svg.png" decoding="async" width="220" height="98" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/5/5f/Gated_Recurrent_Unit.svg/330px-Gated_Recurrent_Unit.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/5/5f/Gated_Recurrent_Unit.svg/440px-Gated_Recurrent_Unit.svg.png 2x" data-file-width="1594" data-file-height="709" /></a><figcaption>Gated recurrent unit</figcaption></figure> Gated recurrent unit (GRU), introduced in 2014, was designed as a simplification of LSTM. They are used in the full form and several further simplified variants.<a href="#cite_note-59">[59]</a><a href="#cite_note-60">[60]</a> They have fewer parameters than LSTM, as they lack an output gate.<a href="#cite_note-MyUser_Wildml.com_May_18_2016c-61">[61]</a> Their performance on polyphonic music modeling and speech signal modeling was found to be similar to that of long short-term memory.<a href="#cite_note-MyUser_Arxiv.org_May_18_2016c2-62">[62]</a> There does not appear to be particular performance difference between LSTM and GRU.<a href="#cite_note-MyUser_Arxiv.org_May_18_2016c2-62">[62]</a><a href="#cite_note-gruber_jockisch-63">[63]</a> <div class="mw-heading mw-heading4"><h4 id="Bidirectional_associative_memory">Bidirectional associative memory</h4>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=16" title="Edit section: Bidirectional associative memory">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Bidirectional_associative_memory" title="Bidirectional associative memory">Bidirectional associative memory</a></div> Introduced by Bart Kosko,<a href="#cite_note-64">[64]</a> a bidirectional associative memory (BAM) network is a variant of a Hopfield network that stores associative data as a vector. The bidirectionality comes from passing information through a matrix and its <a href="/wiki/Transpose" title="Transpose">transpose</a>. Typically, bipolar encoding is preferred to binary encoding of the associative pairs. Recently, stochastic BAM models using <a href="/wiki/Markov_chain" title="Markov chain">Markov</a> stepping were optimized for increased network stability and relevance to real-world applications.<a href="#cite_note-65">[65]</a> A BAM network has two layers, either of which can be driven as an input to recall an association and produce an output on the other layer.<a href="#cite_note-66">[66]</a> <div class="mw-heading mw-heading3"><h3 id="Echo_state">Echo state</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=17" title="Edit section: Echo state">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Echo_state_network" title="Echo state network">Echo state network</a></div> <a href="/wiki/Echo_state_network" title="Echo state network">Echo state networks</a> (ESN) have a sparsely connected random hidden layer. The weights of output neurons are the only part of the network that can change (be trained). ESNs are good at reproducing certain <a href="/wiki/Time_series" title="Time series">time series</a>.<a href="#cite_note-67">[67]</a> A variant for <a href="/wiki/Spiking_neural_network" title="Spiking neural network">spiking neurons</a> is known as a <a href="/wiki/Liquid_state_machine" title="Liquid state machine">liquid state machine</a>.<a href="#cite_note-68">[68]</a> <div class="mw-heading mw-heading3"><h3 id="Recursive">Recursive</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=18" title="Edit section: Recursive">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Recursive_neural_network" title="Recursive neural network">Recursive neural network</a></div> A <a href="/wiki/Recursive_neural_network" title="Recursive neural network">recursive neural network</a><a href="#cite_note-69">[69]</a> is created by applying the same set of weights <a href="/wiki/Recursion" title="Recursion">recursively</a> over a differentiable graph-like structure by traversing the structure in <a href="/wiki/Topological_sort" class="mw-redirect" title="Topological sort">topological order</a>. Such networks are typically also trained by the reverse mode of <a href="/wiki/Automatic_differentiation" title="Automatic differentiation">automatic differentiation</a>.<a href="#cite_note-lin1970-70">[70]</a><a href="#cite_note-grie2008-71">[71]</a> They can process <a href="/wiki/Distributed_representation" class="mw-redirect" title="Distributed representation">distributed representations</a> of structure, such as <a href="/wiki/Mathematical_logic" title="Mathematical logic">logical terms</a>. A special case of recursive neural networks is the RNN whose structure corresponds to a linear chain. Recursive neural networks have been applied to <a href="/wiki/Natural_language_processing" title="Natural language processing">natural language processing</a>.<a href="#cite_note-72">[72]</a> The Recursive Neural Tensor Network uses a <a href="/wiki/Tensor" title="Tensor">tensor</a>-based composition function for all nodes in the tree.<a href="#cite_note-73">[73]</a> <div class="mw-heading mw-heading3"><h3 id="Neural_Turing_machines">Neural Turing machines</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=19" title="Edit section: Neural Turing machines">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main articles: <a href="/wiki/Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a> and <a href="/wiki/Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></div> Neural Turing machines (NTMs) are a method of extending recurrent neural networks by coupling them to external <a href="/wiki/Memory" title="Memory">memory</a> resources with which they interact. The combined system is analogous to a <a href="/wiki/Turing_machine" title="Turing machine">Turing machine</a> or <a href="/wiki/Von_Neumann_architecture" title="Von Neumann architecture">Von Neumann architecture</a> but is <a href="/wiki/Differentiable_neural_computer" title="Differentiable neural computer">differentiable</a> end-to-end, allowing it to be efficiently trained with <a href="/wiki/Gradient_descent" title="Gradient descent">gradient descent</a>.<a href="#cite_note-74">[74]</a> Differentiable neural computers (DNCs) are an extension of Neural Turing machines, allowing for the usage of fuzzy amounts of each memory address and a record of chronology.<a href="#cite_note-DNCnature2016-75">[75]</a> Neural network pushdown automata (NNPDA) are similar to NTMs, but tapes are replaced by analog stacks that are differentiable and trained. In this way, they are similar in complexity to recognizers of <a href="/wiki/Context_free_grammar" class="mw-redirect" title="Context free grammar">context free grammars</a> (CFGs).<a href="#cite_note-76">[76]</a> Recurrent neural networks are <a href="/wiki/Turing_complete" class="mw-redirect" title="Turing complete">Turing complete</a> and can run arbitrary programs to process arbitrary sequences of inputs.<a href="#cite_note-77">[77]</a> <div class="mw-heading mw-heading2"><h2 id="Training">Training</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=20" title="Edit section: Training">edit</a>]</div> <div class="mw-heading mw-heading3"><h3 id="Teacher_forcing">Teacher forcing</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=21" title="Edit section: Teacher forcing">edit</a>]</div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Seq2seq_RNN_encoder-decoder_with_attention_mechanism,_training_and_inferring.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/7/72/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png/220px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png" decoding="async" width="220" height="97" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/7/72/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png/330px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/7/72/Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png/440px-Seq2seq_RNN_encoder-decoder_with_attention_mechanism%2C_training_and_inferring.png 2x" data-file-width="1426" data-file-height="630" /></a><figcaption>Encoder-decoder RNN without attention mechanism. Teacher forcing is shown in red.</figcaption></figure>An RNN can be trained into a conditionally <a href="/wiki/Generative_model" title="Generative model">generative model</a> of sequences, aka autoregression. Concretely, let us consider the problem of machine translation, that is, given a sequence <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x_{1},x_{2},\dots ,x_{n})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (x_{1},x_{2},\dots ,x_{n})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7c0e6a4f6008f01547bb5cc1e8b01207272939e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.337ex; height:2.843ex;" alt="{\displaystyle (x_{1},x_{2},\dots ,x_{n})}"> of English words, the model is to produce a sequence <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (y_{1},\dots ,y_{m})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>m</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (y_{1},\dots ,y_{m})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1f788b82def8d7a3933ae95cc81463bde894ae57" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.995ex; height:2.843ex;" alt="{\displaystyle (y_{1},\dots ,y_{m})}"> of French words. It is to be solved by a <a href="/wiki/Seq2seq" title="Seq2seq">seq2seq</a> model. Now, during training, the encoder half of the model would first ingest <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x_{1},x_{2},\dots ,x_{n})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (x_{1},x_{2},\dots ,x_{n})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7c0e6a4f6008f01547bb5cc1e8b01207272939e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.337ex; height:2.843ex;" alt="{\displaystyle (x_{1},x_{2},\dots ,x_{n})}">, then the decoder half would start generating a sequence <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle ({\hat {y}}_{1},{\hat {y}}_{2},\dots ,{\hat {y}}_{l})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>l</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle ({\hat {y}}_{1},{\hat {y}}_{2},\dots ,{\hat {y}}_{l})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1c690f99611b8115d603410038d1c900eb099b69" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.759ex; height:2.843ex;" alt="{\displaystyle ({\hat {y}}_{1},{\hat {y}}_{2},\dots ,{\hat {y}}_{l})}">. The problem is that if the model makes a mistake early on, say at <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y}}_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\hat {y}}_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2d678e9549266e3dfc09066e1145e3f7a13fd7d2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.356ex; height:2.676ex;" alt="{\displaystyle {\hat {y}}_{2}}">, then subsequent tokens are likely to also be mistakes. This makes it inefficient for the model to obtain a learning signal, since the model would mostly learn to shift <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y}}_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\hat {y}}_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2d678e9549266e3dfc09066e1145e3f7a13fd7d2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.356ex; height:2.676ex;" alt="{\displaystyle {\hat {y}}_{2}}"> towards <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7377c7399e662562cd420fa5c7ce49cfba574998" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.193ex; height:2.009ex;" alt="{\displaystyle y_{2}}">, but not the others. Teacher forcing makes it so that the decoder uses the correct output sequence for generating the next entry in the sequence. So for example, it would see <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (y_{1},\dots ,y_{k})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (y_{1},\dots ,y_{k})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/06fbc6292b20008e7ee5b70877d89c445b81f6d5" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.409ex; height:2.843ex;" alt="{\displaystyle (y_{1},\dots ,y_{k})}"> in order to generate <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y}}_{k+1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\hat {y}}_{k+1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a1af03c401fbfdd242dd870b01707b2d946b0d68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.491ex; height:2.843ex;" alt="{\displaystyle {\hat {y}}_{k+1}}">. <div class="mw-heading mw-heading3"><h3 id="Gradient_descent">Gradient descent</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=22" title="Edit section: Gradient descent">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main articles: <a href="/wiki/Gradient_descent" title="Gradient descent">Gradient descent</a> and <a href="/wiki/Vanishing_gradient_problem" title="Vanishing gradient problem">Vanishing gradient problem</a></div> Gradient descent is a <a href="/wiki/Category:First_order_methods" title="Category:First order methods">first-order</a> <a href="/wiki/Iterative_algorithm" class="mw-redirect" title="Iterative algorithm">iterative</a> <a href="/wiki/Mathematical_optimization" title="Mathematical optimization">optimization</a> <a href="/wiki/Algorithm" title="Algorithm">algorithm</a> for finding the minimum of a function. In neural networks, it can be used to minimize the error term by changing each weight in proportion to the derivative of the error with respect to that weight, provided the non-linear <a href="/wiki/Activation_function" title="Activation function">activation functions</a> are <a href="/wiki/Differentiable_function" title="Differentiable function">differentiable</a>. The standard method for training RNN by gradient descent is the "<a href="/wiki/Backpropagation_through_time" title="Backpropagation through time">backpropagation through time</a>" (BPTT) algorithm, which is a special case of the general algorithm of <a href="/wiki/Backpropagation" title="Backpropagation">backpropagation</a>. A more computationally expensive online variant is called "Real-Time Recurrent Learning" or RTRL,<a href="#cite_note-78">[78]</a><a href="#cite_note-79">[79]</a> which is an instance of <a href="/wiki/Automatic_differentiation" title="Automatic differentiation">automatic differentiation</a> in the forward accumulation mode with stacked tangent vectors. Unlike BPTT, this algorithm is local in time but not local in space. In this context, local in space means that a unit's weight vector can be updated using only information stored in the connected units and the unit itself such that update complexity of a single unit is linear in the dimensionality of the weight vector. Local in time means that the updates take place continually (on-line) and depend only on the most recent time step rather than on multiple time steps within a given time horizon as in BPTT. Biological neural networks appear to be local with respect to both time and space.<a href="#cite_note-80">[80]</a><a href="#cite_note-PríncipeEuliano2000-81">[81]</a> For recursively computing the partial derivatives, RTRL has a time-complexity of O(number of hidden x number of weights) per time step for computing the <a href="/wiki/Jacobian_matrix" class="mw-redirect" title="Jacobian matrix">Jacobian matrices</a>, while BPTT only takes O(number of weights) per time step, at the cost of storing all forward activations within the given time horizon.<a href="#cite_note-Ollivier2015-82">[82]</a> An online hybrid between BPTT and RTRL with intermediate complexity exists,<a href="#cite_note-83">[83]</a><a href="#cite_note-84">[84]</a> along with variants for continuous time.<a href="#cite_note-85">[85]</a> A major problem with gradient descent for standard RNN architectures is that <a href="/wiki/Vanishing_gradient_problem" title="Vanishing gradient problem">error gradients vanish</a> exponentially quickly with the size of the time lag between important events.<a href="#cite_note-hochreiter1991-55">[55]</a><a href="#cite_note-HOCH2001-86">[86]</a> LSTM combined with a BPTT/RTRL hybrid learning method attempts to overcome these problems.<a href="#cite_note-lstm-36">[36]</a> This problem is also solved in the independently recurrent neural network (IndRNN)<a href="#cite_note-auto-87">[87]</a> by reducing the context of a neuron to its own past state and the cross-neuron information can then be explored in the following layers. Memories of different ranges including long-term memory can be learned without the gradient vanishing and exploding problem. The on-line algorithm called causal recursive backpropagation (CRBP), implements and combines BPTT and RTRL paradigms for locally recurrent networks.<a href="#cite_note-88">[88]</a> It works with the most general locally recurrent networks. The CRBP algorithm can minimize the global error term. This fact improves the stability of the algorithm, providing a unifying view of gradient calculation techniques for recurrent networks with local feedback. One approach to gradient information computation in RNNs with arbitrary architectures is based on signal-flow graphs diagrammatic derivation.<a href="#cite_note-89">[89]</a> It uses the BPTT batch algorithm, based on Lee's theorem for network sensitivity calculations.<a href="#cite_note-ReferenceA-90">[90]</a> It was proposed by Wan and Beaufays, while its fast online version was proposed by Campolucci, Uncini and Piazza.<a href="#cite_note-ReferenceA-90">[90]</a> <div class="mw-heading mw-heading3"><h3 id="Connectionist_temporal_classification">Connectionist temporal classification</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=23" title="Edit section: Connectionist temporal classification">edit</a>]</div> The <a href="/wiki/Connectionist_temporal_classification" title="Connectionist temporal classification">connectionist temporal classification</a> (CTC)<a href="#cite_note-graves2006-91">[91]</a> is a specialized loss function for training RNNs for sequence modeling problems where the timing is variable.<a href="#cite_note-92">[92]</a> <div class="mw-heading mw-heading3"><h3 id="Global_optimization_methods">Global optimization methods</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=24" title="Edit section: Global optimization methods">edit</a>]</div> Training the weights in a neural network can be modeled as a non-linear <a href="/wiki/Global_optimization" title="Global optimization">global optimization</a> problem. A target function can be formed to evaluate the fitness or error of a particular weight vector as follows: First, the weights in the network are set according to the weight vector. Next, the network is evaluated against the training sequence. Typically, the sum-squared difference between the predictions and the target values specified in the training sequence is used to represent the error of the current weight vector. Arbitrary global optimization techniques may then be used to minimize this target function. The most common global optimization method for training RNNs is <a href="/wiki/Genetic_algorithm" title="Genetic algorithm">genetic algorithms</a>, especially in unstructured networks.<a href="#cite_note-93">[93]</a><a href="#cite_note-94">[94]</a><a href="#cite_note-95">[95]</a> Initially, the genetic algorithm is encoded with the neural network weights in a predefined manner where one gene in the <a href="/wiki/Chromosome_(genetic_algorithm)" title="Chromosome (genetic algorithm)">chromosome</a> represents one weight link. The whole network is represented as a single chromosome. The fitness function is evaluated as follows: <ul><li>Each weight encoded in the chromosome is assigned to the respective weight link of the network.</li> <li>The training set is presented to the network which propagates the input signals forward.</li> <li>The mean-squared error is returned to the fitness function.</li> <li>This function drives the genetic selection process.</li></ul> Many chromosomes make up the population; therefore, many different neural networks are evolved until a stopping criterion is satisfied. A common stopping scheme is: <ul><li>When the neural network has learned a certain percentage of the training data or</li> <li>When the minimum value of the mean-squared-error is satisfied or</li> <li>When the maximum number of training generations has been reached.</li></ul> The fitness function evaluates the stopping criterion as it receives the mean-squared error reciprocal from each network during training. Therefore, the goal of the genetic algorithm is to maximize the fitness function, reducing the mean-squared error. Other global (and/or evolutionary) optimization techniques may be used to seek a good set of weights, such as <a href="/wiki/Simulated_annealing" title="Simulated annealing">simulated annealing</a> or <a href="/wiki/Particle_swarm_optimization" title="Particle swarm optimization">particle swarm optimization</a>. <div class="mw-heading mw-heading2"><h2 id="Other_architectures">Other architectures</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=25" title="Edit section: Other architectures">edit</a>]</div> <div class="mw-heading mw-heading3"><h3 id="Independently_RNN_(IndRNN)">Independently RNN (IndRNN)</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=26" title="Edit section: Independently RNN (IndRNN)">edit</a>]</div> The independently recurrent neural network (IndRNN)<a href="#cite_note-auto-87">[87]</a> addresses the gradient vanishing and exploding problems in the traditional fully connected RNN. Each neuron in one layer only receives its own past state as context information (instead of full connectivity to all other neurons in this layer) and thus neurons are independent of each other's history. The gradient backpropagation can be regulated to avoid gradient vanishing and exploding in order to keep long or short-term memory. The cross-neuron information is explored in the next layers. IndRNN can be robustly trained with non-saturated nonlinear functions such as ReLU. Deep networks can be trained using skip connections. <div class="mw-heading mw-heading3"><h3 id="Neural_history_compressor">Neural history compressor</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=27" title="Edit section: Neural history compressor">edit</a>]</div> The neural history compressor is an unsupervised stack of RNNs.<a href="#cite_note-schmidhuber1992-96">[96]</a> At the input level, it learns to predict its next input from the previous inputs. Only unpredictable inputs of some RNN in the hierarchy become inputs to the next higher level RNN, which therefore recomputes its internal state only rarely. Each higher level RNN thus studies a compressed representation of the information in the RNN below. This is done such that the input sequence can be precisely reconstructed from the representation at the highest level. The system effectively minimizes the description length or the negative <a href="/wiki/Logarithm" title="Logarithm">logarithm</a> of the probability of the data.<a href="#cite_note-scholarpedia2015pre-97">[97]</a> Given a lot of learnable predictability in the incoming data sequence, the highest level RNN can use supervised learning to easily classify even deep sequences with long intervals between important events. It is possible to distill the RNN hierarchy into two RNNs: the "conscious" chunker (higher level) and the "subconscious" automatizer (lower level).<a href="#cite_note-schmidhuber1992-96">[96]</a> Once the chunker has learned to predict and compress inputs that are unpredictable by the automatizer, then the automatizer can be forced in the next learning phase to predict or imitate through additional units the hidden units of the more slowly changing chunker. This makes it easy for the automatizer to learn appropriate, rarely changing memories across long intervals. In turn, this helps the automatizer to make many of its once unpredictable inputs predictable, such that the chunker can focus on the remaining unpredictable events.<a href="#cite_note-schmidhuber1992-96">[96]</a> A <a href="/wiki/Generative_model" title="Generative model">generative model</a> partially overcame the <a href="/wiki/Vanishing_gradient_problem" title="Vanishing gradient problem">vanishing gradient problem</a><a href="#cite_note-hochreiter1991-55">[55]</a> of <a href="/wiki/Automatic_differentiation" title="Automatic differentiation">automatic differentiation</a> or <a href="/wiki/Backpropagation" title="Backpropagation">backpropagation</a> in neural networks in 1992. In 1993, such a system solved a "Very Deep Learning" task that required more than 1000 subsequent layers in an RNN unfolded in time.<a href="#cite_note-schmidhuber1993-34">[34]</a> <div class="mw-heading mw-heading3"><h3 id="Second_order_RNNs">Second order RNNs</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=28" title="Edit section: Second order RNNs">edit</a>]</div> Second-order RNNs use higher order weights <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w{}_{ijk}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <msub> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> <mi>k</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w{}_{ijk}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/847af94eb75e72db4b5604559682ce920572fa69" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.998ex; height:2.343ex;" alt="{\displaystyle w{}_{ijk}}"> instead of the standard <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w{}_{ij}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <msub> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w{}_{ij}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6ef1a6e570e9d6285d37905fd89313b583d2d871" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.141ex; height:2.343ex;" alt="{\displaystyle w{}_{ij}}"> weights, and states can be a product. This allows a direct mapping to a <a href="/wiki/Finite-state_machine" title="Finite-state machine">finite-state machine</a> both in training, stability, and representation.<a href="#cite_note-98">[98]</a><a href="#cite_note-99">[99]</a> Long short-term memory is an example of this but has no such formal mappings or proof of stability. <div class="mw-heading mw-heading3"><h3 id="Hierarchical_recurrent_neural_network">Hierarchical recurrent neural network</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=29" title="Edit section: Hierarchical recurrent neural network">edit</a>]</div> Hierarchical recurrent neural networks (HRNN) connect their neurons in various ways to decompose hierarchical behavior into useful subprograms.<a href="#cite_note-schmidhuber1992-96">[96]</a><a href="#cite_note-100">[100]</a> Such hierarchical structures of cognition are present in theories of memory presented by philosopher <a href="/wiki/Henri_Bergson" title="Henri Bergson">Henri Bergson</a>, whose philosophical views have inspired hierarchical models.<a href="#cite_note-auto1-101">[101]</a> Hierarchical recurrent neural networks are useful in <a href="/wiki/Forecasting" title="Forecasting">forecasting</a>, helping to predict disaggregated inflation components of the <a href="/wiki/Consumer_price_index" title="Consumer price index">consumer price index</a> (CPI). The HRNN model leverages information from higher levels in the CPI hierarchy to enhance lower-level predictions. Evaluation of a substantial dataset from the US CPI-U index demonstrates the superior performance of the HRNN model compared to various established <a href="/wiki/Inflation" title="Inflation">inflation</a> prediction methods.<a href="#cite_note-barkan-102">[102]</a> <div class="mw-heading mw-heading3"><h3 id="Recurrent_multilayer_perceptron_network">Recurrent multilayer perceptron network</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=30" title="Edit section: Recurrent multilayer perceptron network">edit</a>]</div> Generally, a recurrent multilayer perceptron network (RMLP network) consists of cascaded subnetworks, each containing multiple layers of nodes. Each subnetwork is feed-forward except for the last layer, which can have feedback connections. Each of these subnets is connected only by feed-forward connections.<a href="#cite_note-103">[103]</a> <div class="mw-heading mw-heading3"><h3 id="Multiple_timescales_model">Multiple timescales model</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=31" title="Edit section: Multiple timescales model">edit</a>]</div> A multiple timescales recurrent neural network (MTRNN) is a neural-based computational model that can simulate the functional hierarchy of the brain through self-organization depending on the spatial connection between neurons and on distinct types of neuron activities, each with distinct time properties.<a href="#cite_note-104">[104]</a><a href="#cite_note-105">[105]</a> With such varied neuronal activities, continuous sequences of any set of behaviors are segmented into reusable primitives, which in turn are flexibly integrated into diverse sequential behaviors. The biological approval of such a type of hierarchy was discussed in the <a href="/wiki/Memory-prediction_framework" title="Memory-prediction framework">memory-prediction</a> theory of brain function by <a href="/wiki/Jeff_Hawkins" title="Jeff Hawkins">Hawkins</a> in his book <a href="/wiki/On_Intelligence" title="On Intelligence">On Intelligence</a>.[<a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed">citation needed</a>] Such a hierarchy also agrees with theories of memory posited by philosopher <a href="/wiki/Henri_Bergson" title="Henri Bergson">Henri Bergson</a>, which have been incorporated into an MTRNN model.<a href="#cite_note-auto1-101">[101]</a><a href="#cite_note-106">[106]</a> <div class="mw-heading mw-heading3"><h3 id="Memristive_networks">Memristive networks</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=32" title="Edit section: Memristive networks">edit</a>]</div> Greg Snider of <a href="/wiki/HP_Labs" title="HP Labs">HP Labs</a> describes a system of cortical computing with memristive nanodevices.<a href="#cite_note-107">[107]</a> The <a href="/wiki/Memristors" class="mw-redirect" title="Memristors">memristors</a> (memory resistors) are implemented by thin film materials in which the resistance is electrically tuned via the transport of ions or oxygen vacancies within the film. <a href="/wiki/DARPA" title="DARPA">DARPA</a>'s <a href="/wiki/SyNAPSE" title="SyNAPSE">SyNAPSE project</a> has funded IBM Research and HP Labs, in collaboration with the Boston University Department of Cognitive and Neural Systems (CNS), to develop neuromorphic architectures that may be based on memristive systems. <a href="/w/index.php?title=Memristive_networks&action=edit&redlink=1" class="new" title="Memristive networks (page does not exist)">Memristive networks</a> are a particular type of <a href="/wiki/Physical_neural_network" title="Physical neural network">physical neural network</a> that have very similar properties to (Little-)Hopfield networks, as they have continuous dynamics, a limited memory capacity and natural relaxation via the minimization of a function which is asymptotic to the <a href="/wiki/Ising_model" title="Ising model">Ising model</a>. In this sense, the dynamics of a memristive circuit have the advantage compared to a Resistor-Capacitor network to have a more interesting non-linear behavior. From this point of view, engineering analog memristive networks account for a peculiar type of <a href="/wiki/Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">neuromorphic engineering</a> in which the device behavior depends on the circuit wiring or topology. The evolution of these networks can be studied analytically using variations of the <a href="/w/index.php?title=F.Caravelli&action=edit&redlink=1" class="new" title="F.Caravelli (page does not exist)">Caravelli</a>–<a href="/w/index.php?title=F._Traversa&action=edit&redlink=1" class="new" title="F. Traversa (page does not exist)">Traversa</a>–<a href="/wiki/Di_Ventra" class="mw-redirect" title="Di Ventra">Di Ventra</a> equation.<a href="#cite_note-108">[108]</a> <div class="mw-heading mw-heading3"><h3 id="Continuous-time">Continuous-time</h3>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=33" title="Edit section: Continuous-time">edit</a>]</div> A continuous-time recurrent neural network (CTRNN) uses a system of <a href="/wiki/Ordinary_differential_equations" class="mw-redirect" title="Ordinary differential equations">ordinary differential equations</a> to model the effects on a neuron of the incoming inputs. They are typically analyzed by <a href="/wiki/Dynamical_systems_theory" title="Dynamical systems theory">dynamical systems theory</a>. Many RNN models in neuroscience are continuous-time.<a href="#cite_note-:0-16">[16]</a> For a neuron <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>i</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle i}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/add78d8608ad86e54951b8c8bd6c8d8416533d20" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}"> in the network with activation <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/67d30d30b6c2dbe4d6f150d699de040937ecc95f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.939ex; height:2.009ex;" alt="{\displaystyle y_{i}}">, the rate of change of activation is given by: <dl><dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \tau _{i}{\dot {y}}_{i}=-y_{i}+\sum _{j=1}^{n}w_{ji}\sigma (y_{j}-\Theta _{j})+I_{i}(t)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>τ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <mo>−</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>+</mo> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </munderover> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mi>i</mi> </mrow> </msub> <mi>σ</mi> <mo stretchy="false">(</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> <mo>−</mo> <msub> <mi mathvariant="normal">Θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>+</mo> <msub> <mi>I</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>t</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \tau _{i}{\dot {y}}_{i}=-y_{i}+\sum _{j=1}^{n}w_{ji}\sigma (y_{j}-\Theta _{j})+I_{i}(t)}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fe3aece040129e349beb408629229528edbafb24" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:38.545ex; height:7.176ex;" alt="{\displaystyle \tau _{i}{\dot {y}}_{i}=-y_{i}+\sum _{j=1}^{n}w_{ji}\sigma (y_{j}-\Theta _{j})+I_{i}(t)}"></dd></dl> Where: <ul><li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \tau _{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>τ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \tau _{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/814ca8b33360ac3b7db8e9435271b5654175c853" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.816ex; height:2.009ex;" alt="{\displaystyle \tau _{i}}"> : Time constant of <a href="/wiki/Synapse" title="Synapse">postsynaptic</a> node</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/67d30d30b6c2dbe4d6f150d699de040937ecc95f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.939ex; height:2.009ex;" alt="{\displaystyle y_{i}}"> : Activation of postsynaptic node</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {y}}_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>y</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {y}}_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d7413f2eddfa4154ca58ecb3aba013f9e5e18309" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.102ex; height:2.676ex;" alt="{\displaystyle {\dot {y}}_{i}}"> : Rate of change of activation of postsynaptic node</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w{}_{ji}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <msub> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w{}_{ji}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/921dbd2f06de98163dd3eb31b8ca419431f77241" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.141ex; height:2.343ex;" alt="{\displaystyle w{}_{ji}}"> : Weight of connection from pre to postsynaptic node</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma (x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>σ</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \sigma (x)}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ae09ff47b50183fbfd1ea5697c63963ec9eefa20" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.469ex; height:2.843ex;" alt="{\displaystyle \sigma (x)}"> : <a href="/wiki/Sigmoid_function" title="Sigmoid function">Sigmoid</a> of x e.g. <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma (x)=1/(1+e^{-x})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>σ</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>1</mn> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mo stretchy="false">(</mo> <mn>1</mn> <mo>+</mo> <msup> <mi>e</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>−</mo> <mi>x</mi> </mrow> </msup> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \sigma (x)=1/(1+e^{-x})}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2d652eb008d808b8f71210bb3d2fe48a5ee451a7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.239ex; height:3.009ex;" alt="{\displaystyle \sigma (x)=1/(1+e^{-x})}">.</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{j}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{j}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f8df4e372390588acb968986cfc388e50b930b3a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.049ex; height:2.343ex;" alt="{\displaystyle y_{j}}"> : Activation of presynaptic node</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \Theta _{j}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi mathvariant="normal">Θ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \Theta _{j}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/10b9d5bab99a44d98171cce92f04d0e700060512" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.718ex; height:2.843ex;" alt="{\displaystyle \Theta _{j}}"> : Bias of presynaptic node</li> <li><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I_{i}(t)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>I</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>t</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I_{i}(t)}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9452bc33550571d001aa8ba5399b61818c26fb60" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.472ex; height:2.843ex;" alt="{\displaystyle I_{i}(t)}"> : Input (if any) to node</li></ul> CTRNNs have been applied to <a href="/wiki/Evolutionary_robotics" title="Evolutionary robotics">evolutionary robotics</a> where they have been used to address vision,<a href="#cite_note-109">[109]</a> co-operation,<a href="#cite_note-Evolving_communication_without_dedicated_communication_channels-110">[110]</a> and minimal cognitive behaviour.<a href="#cite_note-The_dynamics_of_adaptive_behavior:_A_research_program-111">[111]</a> Note that, by the <a href="/wiki/Shannon_sampling_theorem" class="mw-redirect" title="Shannon sampling theorem">Shannon sampling theorem</a>, discrete-time recurrent neural networks can be viewed as continuous-time recurrent neural networks where the differential equations have transformed into equivalent <a href="/wiki/Difference_equation" class="mw-redirect" title="Difference equation">difference equations</a>.<a href="#cite_note-Sherstinsky-NeurIPS2018-CRACT-3-112">[112]</a> This transformation can be thought of as occurring after the post-synaptic node activation functions <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{i}(t)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>t</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y_{i}(t)}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/20dfaede0e3f6f701b4c82b5ed153148f5e7d0aa" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.588ex; height:2.843ex;" alt="{\displaystyle y_{i}(t)}"> have been low-pass filtered but prior to sampling. They are in fact <a href="/wiki/Recursive_neural_network" title="Recursive neural network">recursive neural networks</a> with a particular structure: that of a linear chain. Whereas recursive neural networks operate on any hierarchical structure, combining child representations into parent representations, recurrent neural networks operate on the linear progression of time, combining the previous time step and a hidden representation into the representation for the current time step. From a time-series perspective, RNNs can appear as nonlinear versions of <a href="/wiki/Finite_impulse_response" title="Finite impulse response">finite impulse response</a> and <a href="/wiki/Infinite_impulse_response" title="Infinite impulse response">infinite impulse response</a> filters and also as a <a href="/wiki/Nonlinear_autoregressive_exogenous_model" title="Nonlinear autoregressive exogenous model">nonlinear autoregressive exogenous model</a> (NARX).<a href="#cite_note-113">[113]</a> RNN has infinite impulse response whereas <a href="/wiki/Convolutional_neural_network" title="Convolutional neural network">convolutional neural networks</a> have <a href="/wiki/Finite_impulse_response" title="Finite impulse response">finite impulse</a> response. Both classes of networks exhibit temporal <a href="/wiki/Dynamic_system" class="mw-redirect" title="Dynamic system">dynamic behavior</a>.<a href="#cite_note-114">[114]</a> A finite impulse recurrent network is a <a href="/wiki/Directed_acyclic_graph" title="Directed acyclic graph">directed acyclic graph</a> that can be unrolled and replaced with a strictly feedforward neural network, while an infinite impulse recurrent network is a <a href="/wiki/Directed_cyclic_graph" class="mw-redirect" title="Directed cyclic graph">directed cyclic graph</a> that cannot be unrolled. The effect of memory-based learning for the recognition of sequences can also be implemented by a more biological-based model which uses the silencing mechanism exhibited in neurons with a relatively high frequency spiking activity.<a href="#cite_note-115">[115]</a> Additional stored states and the storage under direct control by the network can be added to both <a href="/wiki/Infinite_impulse_response" title="Infinite impulse response">infinite-impulse</a> and <a href="/wiki/Finite_impulse_response" title="Finite impulse response">finite-impulse</a> networks. Another network or graph can also replace the storage if that incorporates time delays or has feedback loops. Such controlled states are referred to as gated states or gated memory and are part of <a href="/wiki/Long_short-term_memory" title="Long short-term memory">long short-term memory</a> networks (LSTMs) and <a href="/wiki/Gated_recurrent_unit" title="Gated recurrent unit">gated recurrent units</a>. This is also called Feedback Neural Network (FNN). <div class="mw-heading mw-heading2"><h2 id="Libraries">Libraries</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=34" title="Edit section: Libraries">edit</a>]</div> Modern libraries provide runtime-optimized implementations of the above functionality or allow to speed up the slow loop by <a href="/wiki/Just-in-time_compilation" title="Just-in-time compilation">just-in-time compilation</a>. <ul><li><a href="/wiki/Apache_Singa" class="mw-redirect" title="Apache Singa">Apache Singa</a></li> <li><a href="/wiki/Caffe_(software)" title="Caffe (software)">Caffe</a>: Created by the Berkeley Vision and Learning Center (BVLC). It supports both CPU and GPU. Developed in <a href="/wiki/C%2B%2B" title="C++">C++</a>, and has <a href="/wiki/Python_(programming_language)" title="Python (programming language)">Python</a> and <a href="/wiki/MATLAB" title="MATLAB">MATLAB</a> wrappers.</li> <li><a href="/wiki/Chainer" title="Chainer">Chainer</a>: Fully in Python, production support for CPU, GPU, distributed training.</li> <li><a href="/wiki/Deeplearning4j" title="Deeplearning4j">Deeplearning4j</a>: Deep learning in <a href="/wiki/Java_(programming_language)" title="Java (programming language)">Java</a> and <a href="/wiki/Scala_(programming_language)" title="Scala (programming language)">Scala</a> on multi-GPU-enabled <a href="/wiki/Apache_Spark" title="Apache Spark">Spark</a>.</li> <li><a href="/wiki/Flux_(machine-learning_framework)" title="Flux (machine-learning framework)">Flux</a>: includes interfaces for RNNs, including GRUs and LSTMs, written in <a href="/wiki/Julia_(programming_language)" title="Julia (programming language)">Julia</a>.</li> <li><a href="/wiki/Keras" title="Keras">Keras</a>: High-level API, providing a wrapper to many other deep learning libraries.</li> <li><a href="/wiki/Microsoft_Cognitive_Toolkit" title="Microsoft Cognitive Toolkit">Microsoft Cognitive Toolkit</a></li> <li><a href="/wiki/MXNet" class="mw-redirect" title="MXNet">MXNet</a>: an open-source deep learning framework used to train and deploy deep neural networks.</li> <li><a href="/wiki/PyTorch" title="PyTorch">PyTorch</a>: Tensors and Dynamic neural networks in Python with GPU acceleration.</li> <li><a href="/wiki/TensorFlow" title="TensorFlow">TensorFlow</a>: Apache 2.0-licensed Theano-like library with support for CPU, GPU and Google's proprietary <a href="/wiki/Tensor_processing_unit" class="mw-redirect" title="Tensor processing unit">TPU</a>,<a href="#cite_note-116">[116]</a> mobile</li> <li><a href="/wiki/Theano_(software)" title="Theano (software)">Theano</a>: A deep-learning library for Python with an API largely compatible with the <a href="/wiki/NumPy" title="NumPy">NumPy</a> library.</li> <li><a href="/wiki/Torch_(machine_learning)" title="Torch (machine learning)">Torch</a>: A scientific computing framework with support for machine learning algorithms, written in <a href="/wiki/C_(programming_language)" title="C (programming language)">C</a> and <a href="/wiki/Lua_(programming_language)" title="Lua (programming language)">Lua</a>.</li></ul> <div class="mw-heading mw-heading2"><h2 id="Applications">Applications</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=35" title="Edit section: Applications">edit</a>]</div> Applications of recurrent neural networks include: <ul><li><a href="/wiki/Machine_translation" title="Machine translation">Machine translation</a><a href="#cite_note-sutskever2014-42">[42]</a></li> <li><a href="/wiki/Robot_control" title="Robot control">Robot control</a><a href="#cite_note-117">[117]</a></li> <li><a href="/wiki/Time_series_prediction" class="mw-redirect" title="Time series prediction">Time series prediction</a><a href="#cite_note-118">[118]</a><a href="#cite_note-119">[119]</a><a href="#cite_note-120">[120]</a></li> <li><a href="/wiki/Speech_recognition" title="Speech recognition">Speech recognition</a><a href="#cite_note-121">[121]</a><a href="#cite_note-fernandez2007keyword-39">[39]</a><a href="#cite_note-graves2013-122">[122]</a></li> <li><a href="/wiki/Speech_synthesis" title="Speech synthesis">Speech synthesis</a><a href="#cite_note-123">[123]</a></li> <li><a href="/wiki/Brain%E2%80%93computer_interfaces" class="mw-redirect" title="Brain–computer interfaces">Brain–computer interfaces</a><a href="#cite_note-124">[124]</a></li> <li>Time series anomaly detection<a href="#cite_note-125">[125]</a></li> <li><a href="/wiki/Text-to-Video_model" class="mw-redirect" title="Text-to-Video model">Text-to-Video model</a><a href="#cite_note-126">[126]</a></li> <li>Rhythm learning<a href="#cite_note-peephole2002-127">[127]</a></li> <li>Music composition<a href="#cite_note-128">[128]</a></li> <li>Grammar learning<a href="#cite_note-129">[129]</a><a href="#cite_note-peepholeLSTM-58">[58]</a><a href="#cite_note-130">[130]</a></li> <li><a href="/wiki/Handwriting_recognition" title="Handwriting recognition">Handwriting recognition</a><a href="#cite_note-131">[131]</a><a href="#cite_note-132">[132]</a></li> <li>Human action recognition<a href="#cite_note-133">[133]</a></li> <li>Protein homology detection<a href="#cite_note-134">[134]</a></li> <li>Predicting subcellular localization of proteins<a href="#cite_note-ThireoReczko-135">[135]</a></li> <li>Several prediction tasks in the area of business process management<a href="#cite_note-136">[136]</a></li> <li>Prediction in medical care pathways<a href="#cite_note-137">[137]</a></li> <li>Predictions of fusion plasma disruptions in reactors (Fusion Recurrent Neural Network (FRNN) code) <a href="#cite_note-138">[138]</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="References">References</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=36" title="Edit section: References">edit</a>]</div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em;"> <ol class="references"> <li id="cite_note-1"><a href="#cite_ref-1">^</a> <style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite id="CITEREFTealab2018" class="citation journal cs1">Tealab, Ahmed (2018-12-01). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.fcij.2018.10.003">"Time series forecasting using artificial neural networks methodologies: A systematic review"</a>. Future Computing and Informatics Journal. 3 (2): 334–340. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.fcij.2018.10.003">10.1016/j.fcij.2018.10.003</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2314-7288">2314-7288</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Future+Computing+and+Informatics+Journal&rft.atitle=Time+series+forecasting+using+artificial+neural+networks+methodologies%3A+A+systematic+review&rft.volume=3&rft.issue=2&rft.pages=334-340&rft.date=2018-12-01&rft_id=info%3Adoi%2F10.1016%2Fj.fcij.2018.10.003&rft.issn=2314-7288&rft.aulast=Tealab&rft.aufirst=Ahmed&rft_id=https%3A%2F%2Fdoi.org%2F10.1016%252Fj.fcij.2018.10.003&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-2"><a href="#cite_ref-2">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesLiwickiFernandezBertolami2009" class="citation journal cs1"><a href="/wiki/Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Graves, Alex</a>; Liwicki, Marcus; Fernandez, Santiago; Bertolami, Roman; Bunke, Horst; <a href="/wiki/J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Schmidhuber, Jürgen</a> (2009). <a rel="nofollow" class="external text" href="http://www.idsia.ch/~juergen/tpami_2008.pdf">"A Novel Connectionist System for Improved Unconstrained Handwriting Recognition"</a> (PDF). IEEE Transactions on Pattern Analysis and Machine Intelligence. 31 (5): 855–868. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.139.4502">10.1.1.139.4502</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2Ftpami.2008.137">10.1109/tpami.2008.137</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/19299860">19299860</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:14635907">14635907</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Pattern+Analysis+and+Machine+Intelligence&rft.atitle=A+Novel+Connectionist+System+for+Improved+Unconstrained+Handwriting+Recognition&rft.volume=31&rft.issue=5&rft.pages=855-868&rft.date=2009&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.139.4502%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A14635907%23id-name%3DS2CID&rft_id=info%3Apmid%2F19299860&rft_id=info%3Adoi%2F10.1109%2Ftpami.2008.137&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Liwicki%2C+Marcus&rft.au=Fernandez%2C+Santiago&rft.au=Bertolami%2C+Roman&rft.au=Bunke%2C+Horst&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fwww.idsia.ch%2F~juergen%2Ftpami_2008.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-sak2014-3">^ <a href="#cite_ref-sak2014_3-0">a</a> <a href="#cite_ref-sak2014_3-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSakSeniorBeaufays2014" class="citation web cs1">Sak, Haşim; Senior, Andrew; Beaufays, Françoise (2014). <a rel="nofollow" class="external text" href="https://research.google.com/pubs/archive/43905.pdf">"Long Short-Term Memory recurrent neural network architectures for large scale acoustic modeling"</a> (PDF). Google Research.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Long+Short-Term+Memory+recurrent+neural+network+architectures+for+large+scale+acoustic+modeling&rft.pub=Google+Research&rft.date=2014&rft.aulast=Sak&rft.aufirst=Ha%C5%9Fim&rft.au=Senior%2C+Andrew&rft.au=Beaufays%2C+Fran%C3%A7oise&rft_id=https%3A%2F%2Fresearch.google.com%2Fpubs%2Farchive%2F43905.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-liwu2015-4">^ <a href="#cite_ref-liwu2015_4-0">a</a> <a href="#cite_ref-liwu2015_4-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLiWu2014" class="citation arxiv cs1">Li, Xiangang; Wu, Xihong (2014-10-15). "Constructing Long Short-Term Memory based Deep Recurrent Neural Networks for Large Vocabulary Speech Recognition". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1410.4281">1410.4281</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Constructing+Long+Short-Term+Memory+based+Deep+Recurrent+Neural+Networks+for+Large+Vocabulary+Speech+Recognition&rft.date=2014-10-15&rft_id=info%3Aarxiv%2F1410.4281&rft.aulast=Li&rft.aufirst=Xiangang&rft.au=Wu%2C+Xihong&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-5"><a href="#cite_ref-5">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDupond2019" class="citation journal cs1">Dupond, Samuel (2019). <a rel="nofollow" class="external text" href="https://www.sciencedirect.com/journal/annual-reviews-in-control">"A thorough review on the current advance of neural network structures"</a>. Annual Reviews in Control. 14: 200–230.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Annual+Reviews+in+Control&rft.atitle=A+thorough+review+on+the+current+advance+of+neural+network+structures.&rft.volume=14&rft.pages=200-230&rft.date=2019&rft.aulast=Dupond&rft.aufirst=Samuel&rft_id=https%3A%2F%2Fwww.sciencedirect.com%2Fjournal%2Fannual-reviews-in-control&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-6"><a href="#cite_ref-6">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFAbiodunJantanOmolaraDada2018" class="citation journal cs1">Abiodun, Oludare Isaac; Jantan, Aman; Omolara, Abiodun Esther; Dada, Kemi Victoria; Mohamed, Nachaat Abdelatif; Arshad, Humaira (2018-11-01). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6260436">"State-of-the-art in artificial neural network applications: A survey"</a>. Heliyon. 4 (11): e00938. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2018Heliy...400938A">2018Heliy...400938A</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.heliyon.2018.e00938">10.1016/j.heliyon.2018.e00938</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2405-8440">2405-8440</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6260436">6260436</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/30519653">30519653</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Heliyon&rft.atitle=State-of-the-art+in+artificial+neural+network+applications%3A+A+survey&rft.volume=4&rft.issue=11&rft.pages=e00938&rft.date=2018-11-01&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6260436%23id-name%3DPMC&rft_id=info%3Abibcode%2F2018Heliy...400938A&rft_id=info%3Apmid%2F30519653&rft_id=info%3Adoi%2F10.1016%2Fj.heliyon.2018.e00938&rft.issn=2405-8440&rft.aulast=Abiodun&rft.aufirst=Oludare+Isaac&rft.au=Jantan%2C+Aman&rft.au=Omolara%2C+Abiodun+Esther&rft.au=Dada%2C+Kemi+Victoria&rft.au=Mohamed%2C+Nachaat+Abdelatif&rft.au=Arshad%2C+Humaira&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6260436&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-7"><a href="#cite_ref-7">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFEspinosa-SanchezGomez-Marinde_Castro2023" class="citation journal cs1">Espinosa-Sanchez, Juan Manuel; Gomez-Marin, Alex; de Castro, Fernando (2023-07-05). <a rel="nofollow" class="external text" href="http://journals.sagepub.com/doi/10.1177/10738584231179932">"The Importance of Cajal's and Lorente de Nó's Neuroscience to the Birth of Cybernetics"</a>. The Neuroscientist. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F10738584231179932">10.1177/10738584231179932</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/10261%2F348372">10261/348372</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1073-8584">1073-8584</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/37403768">37403768</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Neuroscientist&rft.atitle=The+Importance+of+Cajal%27s+and+Lorente+de+N%C3%B3%27s+Neuroscience+to+the+Birth+of+Cybernetics&rft.date=2023-07-05&rft_id=info%3Ahdl%2F10261%2F348372&rft.issn=1073-8584&rft_id=info%3Apmid%2F37403768&rft_id=info%3Adoi%2F10.1177%2F10738584231179932&rft.aulast=Espinosa-Sanchez&rft.aufirst=Juan+Manuel&rft.au=Gomez-Marin%2C+Alex&rft.au=de+Castro%2C+Fernando&rft_id=http%3A%2F%2Fjournals.sagepub.com%2Fdoi%2F10.1177%2F10738584231179932&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-8"><a href="#cite_ref-8">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRamón_y_Cajal1909" class="citation book cs1">Ramón y Cajal, Santiago (1909). <a rel="nofollow" class="external text" href="https://archive.org/details/b2129592x_0002/page/n159/mode/2up">Histologie du système nerveux de l'homme & des vertébrés</a>. Vol. II. Foyle Special Collections Library King's College London. Paris : A. Maloine. p. 149.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Histologie+du+syst%C3%A8me+nerveux+de+l%27homme+%26+des+vert%C3%A9br%C3%A9s&rft.pages=149&rft.pub=Paris+%3A+A.+Maloine&rft.date=1909&rft.aulast=Ram%C3%B3n+y+Cajal&rft.aufirst=Santiago&rft_id=https%3A%2F%2Farchive.org%2Fdetails%2Fb2129592x_0002%2Fpage%2Fn159%2Fmode%2F2up&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-9"><a href="#cite_ref-9">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFde_NÓ1933" class="citation journal cs1">de NÓ, R. Lorente (1933-08-01). <a rel="nofollow" class="external text" href="http://archneurpsyc.jamanetwork.com/article.aspx?doi=10.1001/archneurpsyc.1933.02240140009001">"Vestibulo-Ocular Reflex Arc"</a>. Archives of Neurology and Psychiatry. 30 (2): 245. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1001%2Farchneurpsyc.1933.02240140009001">10.1001/archneurpsyc.1933.02240140009001</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0096-6754">0096-6754</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Archives+of+Neurology+and+Psychiatry&rft.atitle=Vestibulo-Ocular+Reflex+Arc&rft.volume=30&rft.issue=2&rft.pages=245&rft.date=1933-08-01&rft_id=info%3Adoi%2F10.1001%2Farchneurpsyc.1933.02240140009001&rft.issn=0096-6754&rft.aulast=de+N%C3%93&rft.aufirst=R.+Lorente&rft_id=http%3A%2F%2Farchneurpsyc.jamanetwork.com%2Farticle.aspx%3Fdoi%3D10.1001%2Farchneurpsyc.1933.02240140009001&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-10"><a href="#cite_ref-10">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLarriva-Sahd2014" class="citation journal cs1">Larriva-Sahd, Jorge A. (2014-12-03). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4253658">"Some predictions of Rafael Lorente de Nó 80 years later"</a>. Frontiers in Neuroanatomy. 8: 147. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.3389%2Ffnana.2014.00147">10.3389/fnana.2014.00147</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1662-5129">1662-5129</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4253658">4253658</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/25520630">25520630</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Frontiers+in+Neuroanatomy&rft.atitle=Some+predictions+of+Rafael+Lorente+de+N%C3%B3+80+years+later&rft.volume=8&rft.pages=147&rft.date=2014-12-03&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC4253658%23id-name%3DPMC&rft.issn=1662-5129&rft_id=info%3Apmid%2F25520630&rft_id=info%3Adoi%2F10.3389%2Ffnana.2014.00147&rft.aulast=Larriva-Sahd&rft.aufirst=Jorge+A.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC4253658&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-11"><a href="#cite_ref-11">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.oxfordreference.com/display/10.1093/oi/authority.20110803100417461">"reverberating circuit"</a>. Oxford Reference. Retrieved 2024-07-27.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=Oxford+Reference&rft.atitle=reverberating+circuit&rft_id=https%3A%2F%2Fwww.oxfordreference.com%2Fdisplay%2F10.1093%2Foi%2Fauthority.20110803100417461&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-12"><a href="#cite_ref-12">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMcCullochPitts1943" class="citation journal cs1">McCulloch, Warren S.; Pitts, Walter (December 1943). <a rel="nofollow" class="external text" href="http://link.springer.com/10.1007/BF02478259">"A logical calculus of the ideas immanent in nervous activity"</a>. The Bulletin of Mathematical Biophysics. 5 (4): 115–133. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2FBF02478259">10.1007/BF02478259</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0007-4985">0007-4985</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Bulletin+of+Mathematical+Biophysics&rft.atitle=A+logical+calculus+of+the+ideas+immanent+in+nervous+activity&rft.volume=5&rft.issue=4&rft.pages=115-133&rft.date=1943-12&rft_id=info%3Adoi%2F10.1007%2FBF02478259&rft.issn=0007-4985&rft.aulast=McCulloch&rft.aufirst=Warren+S.&rft.au=Pitts%2C+Walter&rft_id=http%3A%2F%2Flink.springer.com%2F10.1007%2FBF02478259&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-13"><a href="#cite_ref-13">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMoreno-DíazMoreno-Díaz2007" class="citation journal cs1">Moreno-Díaz, Roberto; Moreno-Díaz, Arminda (April 2007). <a rel="nofollow" class="external text" href="https://linkinghub.elsevier.com/retrieve/pii/S0303264706002152">"On the legacy of W.S. McCulloch"</a>. Biosystems. 88 (3): 185–190. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2007BiSys..88..185M">2007BiSys..88..185M</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.biosystems.2006.08.010">10.1016/j.biosystems.2006.08.010</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/17184902">17184902</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Biosystems&rft.atitle=On+the+legacy+of+W.S.+McCulloch&rft.volume=88&rft.issue=3&rft.pages=185-190&rft.date=2007-04&rft_id=info%3Apmid%2F17184902&rft_id=info%3Adoi%2F10.1016%2Fj.biosystems.2006.08.010&rft_id=info%3Abibcode%2F2007BiSys..88..185M&rft.aulast=Moreno-D%C3%ADaz&rft.aufirst=Roberto&rft.au=Moreno-D%C3%ADaz%2C+Arminda&rft_id=https%3A%2F%2Flinkinghub.elsevier.com%2Fretrieve%2Fpii%2FS0303264706002152&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-14"><a href="#cite_ref-14">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFArbib2000" class="citation journal cs1">Arbib, Michael A (December 2000). <a rel="nofollow" class="external text" href="https://muse.jhu.edu/article/46496">"Warren McCulloch's Search for the Logic of the Nervous System"</a>. Perspectives in Biology and Medicine. 43 (2): 193–216. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1353%2Fpbm.2000.0001">10.1353/pbm.2000.0001</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1529-8795">1529-8795</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/10804585">10804585</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Perspectives+in+Biology+and+Medicine&rft.atitle=Warren+McCulloch%27s+Search+for+the+Logic+of+the+Nervous+System&rft.volume=43&rft.issue=2&rft.pages=193-216&rft.date=2000-12&rft.issn=1529-8795&rft_id=info%3Apmid%2F10804585&rft_id=info%3Adoi%2F10.1353%2Fpbm.2000.0001&rft.aulast=Arbib&rft.aufirst=Michael+A&rft_id=https%3A%2F%2Fmuse.jhu.edu%2Farticle%2F46496&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-15"><a href="#cite_ref-15">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRenshaw1946" class="citation journal cs1">Renshaw, Birdsey (1946-05-01). <a rel="nofollow" class="external text" href="https://www.physiology.org/doi/10.1152/jn.1946.9.3.191">"Central Effects of Centripetal Impulses in Axons of Spinal Ventral Roots"</a>. Journal of Neurophysiology. 9 (3): 191–204. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1152%2Fjn.1946.9.3.191">10.1152/jn.1946.9.3.191</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0022-3077">0022-3077</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/21028162">21028162</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Neurophysiology&rft.atitle=Central+Effects+of+Centripetal+Impulses+in+Axons+of+Spinal+Ventral+Roots&rft.volume=9&rft.issue=3&rft.pages=191-204&rft.date=1946-05-01&rft.issn=0022-3077&rft_id=info%3Apmid%2F21028162&rft_id=info%3Adoi%2F10.1152%2Fjn.1946.9.3.191&rft.aulast=Renshaw&rft.aufirst=Birdsey&rft_id=https%3A%2F%2Fwww.physiology.org%2Fdoi%2F10.1152%2Fjn.1946.9.3.191&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-:0-16">^ <a href="#cite_ref-:0_16-0">a</a> <a href="#cite_ref-:0_16-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGrossberg2013" class="citation journal cs1">Grossberg, Stephen (2013-02-22). <a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.1888">"Recurrent Neural Networks"</a>. Scholarpedia. 8 (2): 1888. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2013SchpJ...8.1888G">2013SchpJ...8.1888G</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.1888">10.4249/scholarpedia.1888</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1941-6016">1941-6016</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Scholarpedia&rft.atitle=Recurrent+Neural+Networks&rft.volume=8&rft.issue=2&rft.pages=1888&rft.date=2013-02-22&rft.issn=1941-6016&rft_id=info%3Adoi%2F10.4249%2Fscholarpedia.1888&rft_id=info%3Abibcode%2F2013SchpJ...8.1888G&rft.aulast=Grossberg&rft.aufirst=Stephen&rft_id=https%3A%2F%2Fdoi.org%2F10.4249%252Fscholarpedia.1888&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-:1-17">^ <a href="#cite_ref-:1_17-0">a</a> <a href="#cite_ref-:1_17-1">b</a> <a href="#cite_ref-:1_17-2">c</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRosenblatt1961" class="citation book cs1">Rosenblatt, Frank (1961-03-15). <a rel="nofollow" class="external text" href="https://archive.org/details/DTIC_AD0256582/page/n3/mode/2up">DTIC AD0256582: PRINCIPLES OF NEURODYNAMICS. PERCEPTRONS AND THE THEORY OF BRAIN MECHANISMS</a>. Defense Technical Information Center.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=DTIC+AD0256582%3A+PRINCIPLES+OF+NEURODYNAMICS.+PERCEPTRONS+AND+THE+THEORY+OF+BRAIN+MECHANISMS&rft.pub=Defense+Technical+Information+Center&rft.date=1961-03-15&rft.aulast=Rosenblatt&rft.aufirst=Frank&rft_id=https%3A%2F%2Farchive.org%2Fdetails%2FDTIC_AD0256582%2Fpage%2Fn3%2Fmode%2F2up&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-18"><a href="#cite_ref-18">^</a> F. Rosenblatt, "<a href="//archive.org/details/SelfOrganizingSystems/page/n87/mode/1up" class="extiw" title="iarchive:SelfOrganizingSystems/page/n87/mode/1up">Perceptual Generalization over Transformation Groups</a>", pp. 63--100 in Self-organizing Systems: Proceedings of an Inter-disciplinary Conference, 5 and 6 May, 1959. Edited by Marshall C. Yovitz and Scott Cameron. London, New York, [etc.], Pergamon Press, 1960. ix, 322 p. </li> <li id="cite_note-Nakano1971-19"><a href="#cite_ref-Nakano1971_19-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFNakano1971" class="citation book cs1">Nakano, Kaoru (1971). "Learning Process in a Model of Associative Memory". Pattern Recognition and Machine Learning. pp. 172–186. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-1-4615-7566-5_15">10.1007/978-1-4615-7566-5_15</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-4615-7568-9" title="Special:BookSources/978-1-4615-7568-9"><bdi>978-1-4615-7568-9</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Learning+Process+in+a+Model+of+Associative+Memory&rft.btitle=Pattern+Recognition+and+Machine+Learning&rft.pages=172-186&rft.date=1971&rft_id=info%3Adoi%2F10.1007%2F978-1-4615-7566-5_15&rft.isbn=978-1-4615-7568-9&rft.aulast=Nakano&rft.aufirst=Kaoru&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Nakano1972-20"><a href="#cite_ref-Nakano1972_20-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFNakano1972" class="citation journal cs1 cs1-prop-long-vol">Nakano, Kaoru (1972). "Associatron-A Model of Associative Memory". IEEE Transactions on Systems, Man, and Cybernetics. SMC-2 (3): 380–388. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTSMC.1972.4309133">10.1109/TSMC.1972.4309133</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Systems%2C+Man%2C+and+Cybernetics&rft.atitle=Associatron-A+Model+of+Associative+Memory&rft.volume=SMC-2&rft.issue=3&rft.pages=380-388&rft.date=1972&rft_id=info%3Adoi%2F10.1109%2FTSMC.1972.4309133&rft.aulast=Nakano&rft.aufirst=Kaoru&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Amari1972-21"><a href="#cite_ref-Amari1972_21-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFAmari1972" class="citation journal cs1">Amari, Shun-Ichi (1972). "Learning patterns and pattern sequences by self-organizing nets of threshold elements". IEEE Transactions. C (21): 1197–1206.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions&rft.atitle=Learning+patterns+and+pattern+sequences+by+self-organizing+nets+of+threshold+elements&rft.volume=C&rft.issue=21&rft.pages=1197-1206&rft.date=1972&rft.aulast=Amari&rft.aufirst=Shun-Ichi&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-little74-22"><a href="#cite_ref-little74_22-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLittle1974" class="citation journal cs1">Little, W. A. (1974). "The Existence of Persistent States in the Brain". Mathematical Biosciences. 19 (1–2): 101–120. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0025-5564%2874%2990031-5">10.1016/0025-5564(74)90031-5</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Mathematical+Biosciences&rft.atitle=The+Existence+of+Persistent+States+in+the+Brain&rft.volume=19&rft.issue=1%E2%80%932&rft.pages=101-120&rft.date=1974&rft_id=info%3Adoi%2F10.1016%2F0025-5564%2874%2990031-5&rft.aulast=Little&rft.aufirst=W.+A.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-lenz1920-23"><a href="#cite_ref-lenz1920_23-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLenz1920" class="citation cs2"><a href="/wiki/Wilhelm_Lenz" title="Wilhelm Lenz">Lenz, W.</a> (1920), "Beiträge zum Verständnis der magnetischen Eigenschaften in festen Körpern", Physikalische Zeitschrift, 21: 613–615.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physikalische+Zeitschrift&rft.atitle=Beitr%C3%A4ge+zum+Verst%C3%A4ndnis+der+magnetischen+Eigenschaften+in+festen+K%C3%B6rpern&rft.volume=21&rft.pages=613-615&rft.date=1920&rft.aulast=Lenz&rft.aufirst=W.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-ising1925-24"><a href="#cite_ref-ising1925_24-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFIsing1925" class="citation cs2">Ising, E. (1925), "Beitrag zur Theorie des Ferromagnetismus", Z. Phys., 31 (1): 253–258, <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1925ZPhy...31..253I">1925ZPhy...31..253I</a>, <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2FBF02980577">10.1007/BF02980577</a>, <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:122157319">122157319</a></cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Z.+Phys.&rft.atitle=Beitrag+zur+Theorie+des+Ferromagnetismus&rft.volume=31&rft.issue=1&rft.pages=253-258&rft.date=1925&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A122157319%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1007%2FBF02980577&rft_id=info%3Abibcode%2F1925ZPhy...31..253I&rft.aulast=Ising&rft.aufirst=E.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-25"><a href="#cite_ref-25">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBrush1967" class="citation journal cs1">Brush, Stephen G. (1967). "History of the Lenz-Ising Model". Reviews of Modern Physics. 39 (4): 883–893. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1967RvMP...39..883B">1967RvMP...39..883B</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FRevModPhys.39.883">10.1103/RevModPhys.39.883</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Reviews+of+Modern+Physics&rft.atitle=History+of+the+Lenz-Ising+Model&rft.volume=39&rft.issue=4&rft.pages=883-893&rft.date=1967&rft_id=info%3Adoi%2F10.1103%2FRevModPhys.39.883&rft_id=info%3Abibcode%2F1967RvMP...39..883B&rft.aulast=Brush&rft.aufirst=Stephen+G.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-:22-26"><a href="#cite_ref-:22_26-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGlauber1963" class="citation journal cs1">Glauber, Roy J. (February 1963). <a rel="nofollow" class="external text" href="https://aip.scitation.org/doi/abs/10.1063/1.1703954">"Roy J. Glauber "Time-Dependent Statistics of the Ising Model""</a>. Journal of Mathematical Physics. 4 (2): 294–307. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1063%2F1.1703954">10.1063/1.1703954</a>. Retrieved 2021-03-21.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Mathematical+Physics&rft.atitle=Roy+J.+Glauber+%22Time-Dependent+Statistics+of+the+Ising+Model%22&rft.volume=4&rft.issue=2&rft.pages=294-307&rft.date=1963-02&rft_id=info%3Adoi%2F10.1063%2F1.1703954&rft.aulast=Glauber&rft.aufirst=Roy+J.&rft_id=https%3A%2F%2Faip.scitation.org%2Fdoi%2Fabs%2F10.1063%2F1.1703954&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-27"><a href="#cite_ref-27">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSherringtonKirkpatrick1975" class="citation journal cs1">Sherrington, David; Kirkpatrick, Scott (1975-12-29). <a rel="nofollow" class="external text" href="https://link.aps.org/doi/10.1103/PhysRevLett.35.1792">"Solvable Model of a Spin-Glass"</a>. Physical Review Letters. 35 (26): 1792–1796. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1975PhRvL..35.1792S">1975PhRvL..35.1792S</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevLett.35.1792">10.1103/PhysRevLett.35.1792</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0031-9007">0031-9007</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+Letters&rft.atitle=Solvable+Model+of+a+Spin-Glass&rft.volume=35&rft.issue=26&rft.pages=1792-1796&rft.date=1975-12-29&rft.issn=0031-9007&rft_id=info%3Adoi%2F10.1103%2FPhysRevLett.35.1792&rft_id=info%3Abibcode%2F1975PhRvL..35.1792S&rft.aulast=Sherrington&rft.aufirst=David&rft.au=Kirkpatrick%2C+Scott&rft_id=https%3A%2F%2Flink.aps.org%2Fdoi%2F10.1103%2FPhysRevLett.35.1792&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Hopfield19822-28"><a href="#cite_ref-Hopfield19822_28-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHopfield1982" class="citation journal cs1">Hopfield, J. J. (1982). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC346238">"Neural networks and physical systems with emergent collective computational abilities"</a>. Proceedings of the National Academy of Sciences. 79 (8): 2554–2558. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1982PNAS...79.2554H">1982PNAS...79.2554H</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1073%2Fpnas.79.8.2554">10.1073/pnas.79.8.2554</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC346238">346238</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/6953413">6953413</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+the+National+Academy+of+Sciences&rft.atitle=Neural+networks+and+physical+systems+with+emergent+collective+computational+abilities&rft.volume=79&rft.issue=8&rft.pages=2554-2558&rft.date=1982&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC346238%23id-name%3DPMC&rft_id=info%3Apmid%2F6953413&rft_id=info%3Adoi%2F10.1073%2Fpnas.79.8.2554&rft_id=info%3Abibcode%2F1982PNAS...79.2554H&rft.aulast=Hopfield&rft.aufirst=J.+J.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC346238&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-:02-29"><a href="#cite_ref-:02_29-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHopfield1984" class="citation journal cs1">Hopfield, J. J. (1984). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC345226">"Neurons with graded response have collective computational properties like those of two-state neurons"</a>. Proceedings of the National Academy of Sciences. 81 (10): 3088–3092. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1984PNAS...81.3088H">1984PNAS...81.3088H</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1073%2Fpnas.81.10.3088">10.1073/pnas.81.10.3088</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC345226">345226</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/6587342">6587342</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+the+National+Academy+of+Sciences&rft.atitle=Neurons+with+graded+response+have+collective+computational+properties+like+those+of+two-state+neurons&rft.volume=81&rft.issue=10&rft.pages=3088-3092&rft.date=1984&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC345226%23id-name%3DPMC&rft_id=info%3Apmid%2F6587342&rft_id=info%3Adoi%2F10.1073%2Fpnas.81.10.3088&rft_id=info%3Abibcode%2F1984PNAS...81.3088H&rft.aulast=Hopfield&rft.aufirst=J.+J.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC345226&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-30"><a href="#cite_ref-30">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFEngelBroeck2001" class="citation book cs1">Engel, A.; Broeck, C. van den (2001). Statistical mechanics of learning. Cambridge, UK ; New York, NY: Cambridge University Press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-521-77307-2" title="Special:BookSources/978-0-521-77307-2"><bdi>978-0-521-77307-2</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Statistical+mechanics+of+learning&rft.place=Cambridge%2C+UK+%3B+New+York%2C+NY&rft.pub=Cambridge+University+Press&rft.date=2001&rft.isbn=978-0-521-77307-2&rft.aulast=Engel&rft.aufirst=A.&rft.au=Broeck%2C+C.+van+den&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-31"><a href="#cite_ref-31">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSeungSompolinskyTishby1992" class="citation journal cs1">Seung, H. S.; Sompolinsky, H.; Tishby, N. (1992-04-01). <a rel="nofollow" class="external text" href="https://journals.aps.org/pra/abstract/10.1103/PhysRevA.45.6056">"Statistical mechanics of learning from examples"</a>. Physical Review A. 45 (8): 6056–6091. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1992PhRvA..45.6056S">1992PhRvA..45.6056S</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevA.45.6056">10.1103/PhysRevA.45.6056</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/9907706">9907706</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+A&rft.atitle=Statistical+mechanics+of+learning+from+examples&rft.volume=45&rft.issue=8&rft.pages=6056-6091&rft.date=1992-04-01&rft_id=info%3Apmid%2F9907706&rft_id=info%3Adoi%2F10.1103%2FPhysRevA.45.6056&rft_id=info%3Abibcode%2F1992PhRvA..45.6056S&rft.aulast=Seung&rft.aufirst=H.+S.&rft.au=Sompolinsky%2C+H.&rft.au=Tishby%2C+N.&rft_id=https%3A%2F%2Fjournals.aps.org%2Fpra%2Fabstract%2F10.1103%2FPhysRevA.45.6056&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-32"><a href="#cite_ref-32">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFZhangLiptonLiSmola2024" class="citation book cs1">Zhang, Aston; Lipton, Zachary; Li, Mu; Smola, Alexander J. (2024). <a rel="nofollow" class="external text" href="https://d2l.ai/chapter_recurrent-modern/index.html">"10. Modern Recurrent Neural Networks"</a>. Dive into deep learning. Cambridge New York Port Melbourne New Delhi Singapore: Cambridge University Press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-009-38943-3" title="Special:BookSources/978-1-009-38943-3"><bdi>978-1-009-38943-3</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=10.+Modern+Recurrent+Neural+Networks&rft.btitle=Dive+into+deep+learning&rft.place=Cambridge+New+York+Port+Melbourne+New+Delhi+Singapore&rft.pub=Cambridge+University+Press&rft.date=2024&rft.isbn=978-1-009-38943-3&rft.aulast=Zhang&rft.aufirst=Aston&rft.au=Lipton%2C+Zachary&rft.au=Li%2C+Mu&rft.au=Smola%2C+Alexander+J.&rft_id=https%3A%2F%2Fd2l.ai%2Fchapter_recurrent-modern%2Findex.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-33"><a href="#cite_ref-33">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRumelhartHintonWilliams1986" class="citation journal cs1">Rumelhart, David E.; Hinton, Geoffrey E.; Williams, Ronald J. (October 1986). <a rel="nofollow" class="external text" href="https://www.nature.com/articles/323533a0">"Learning representations by back-propagating errors"</a>. Nature. 323 (6088): 533–536. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1986Natur.323..533R">1986Natur.323..533R</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2F323533a0">10.1038/323533a0</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1476-4687">1476-4687</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Nature&rft.atitle=Learning+representations+by+back-propagating+errors&rft.volume=323&rft.issue=6088&rft.pages=533-536&rft.date=1986-10&rft.issn=1476-4687&rft_id=info%3Adoi%2F10.1038%2F323533a0&rft_id=info%3Abibcode%2F1986Natur.323..533R&rft.aulast=Rumelhart&rft.aufirst=David+E.&rft.au=Hinton%2C+Geoffrey+E.&rft.au=Williams%2C+Ronald+J.&rft_id=https%3A%2F%2Fwww.nature.com%2Farticles%2F323533a0&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-schmidhuber1993-34">^ <a href="#cite_ref-schmidhuber1993_34-0">a</a> <a href="#cite_ref-schmidhuber1993_34-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSchmidhuber1993" class="citation book cs1">Schmidhuber, Jürgen (1993). <a rel="nofollow" class="external text" href="ftp://ftp.idsia.ch/pub/juergen/habilitation.pdf">Habilitation thesis: System modeling and optimization</a> (PDF).</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Habilitation+thesis%3A+System+modeling+and+optimization&rft.date=1993&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rft_id=ftp%3A%2F%2Fftp.idsia.ch%2Fpub%2Fjuergen%2Fhabilitation.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988">[<a href="/wiki/Wikipedia:Link_rot" title="Wikipedia:Link rot">permanent dead link</a>‍] Page 150 ff demonstrates credit assignment across the equivalent of 1,200 layers in an unfolded RNN. </li> <li id="cite_note-35"><a href="#cite_ref-35">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSepp_HochreiterJürgen_Schmidhuber1995" class="citation cs2"><a href="/wiki/Sepp_Hochreiter" title="Sepp Hochreiter">Sepp Hochreiter</a>; <a href="/wiki/J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a> (21 August 1995), <a rel="nofollow" class="external text" href="ftp://ftp.idsia.ch/pub/juergen/fki-207-95.ps.gz">Long Short Term Memory</a>, <a href="/wiki/WDQ_(identifier)" class="mw-redirect" title="WDQ (identifier)">Wikidata</a> <a href="https://www.wikidata.org/wiki/Q98967430" class="extiw" title="d:Q98967430">Q98967430</a></cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Long+Short+Term+Memory&rft.date=1995-08-21&rft.au=Sepp+Hochreiter&rft.au=J%C3%BCrgen+Schmidhuber&rft_id=ftp%3A%2F%2Fftp.idsia.ch%2Fpub%2Fjuergen%2Ffki-207-95.ps.gz&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-lstm-36">^ <a href="#cite_ref-lstm_36-0">a</a> <a href="#cite_ref-lstm_36-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHochreiterSchmidhuber1997" class="citation journal cs1"><a href="/wiki/Sepp_Hochreiter" title="Sepp Hochreiter">Hochreiter, Sepp</a>; Schmidhuber, Jürgen (1997-11-01). "Long Short-Term Memory". Neural Computation. 9 (8): 1735–1780. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fneco.1997.9.8.1735">10.1162/neco.1997.9.8.1735</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/9377276">9377276</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1915014">1915014</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Long+Short-Term+Memory&rft.volume=9&rft.issue=8&rft.pages=1735-1780&rft.date=1997-11-01&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1915014%23id-name%3DS2CID&rft_id=info%3Apmid%2F9377276&rft_id=info%3Adoi%2F10.1162%2Fneco.1997.9.8.1735&rft.aulast=Hochreiter&rft.aufirst=Sepp&rft.au=Schmidhuber%2C+J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Schuster-37"><a href="#cite_ref-Schuster_37-0">^</a> Schuster, Mike, and Kuldip K. Paliwal. "<a rel="nofollow" class="external text" href="https://www.researchgate.net/profile/Mike_Schuster/publication/3316656_Bidirectional_recurrent_neural_networks/links/56861d4008ae19758395f85c.pdf">Bidirectional recurrent neural networks</a>." Signal Processing, IEEE Transactions on 45.11 (1997): 2673-2681.2. Awni Hannun, Carl Case, Jared Casper, Bryan Catanzaro, Greg Diamos, Erich Elsen, Ryan </li> <li id="cite_note-38"><a href="#cite_ref-38">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesSchmidhuber2005" class="citation journal cs1">Graves, Alex; Schmidhuber, Jürgen (2005-07-01). "Framewise phoneme classification with bidirectional LSTM and other neural network architectures". Neural Networks. IJCNN 2005. 18 (5): 602–610. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.331.5800">10.1.1.331.5800</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.neunet.2005.06.042">10.1016/j.neunet.2005.06.042</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/16112549">16112549</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1856462">1856462</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Networks&rft.atitle=Framewise+phoneme+classification+with+bidirectional+LSTM+and+other+neural+network+architectures&rft.volume=18&rft.issue=5&rft.pages=602-610&rft.date=2005-07-01&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.331.5800%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1856462%23id-name%3DS2CID&rft_id=info%3Apmid%2F16112549&rft_id=info%3Adoi%2F10.1016%2Fj.neunet.2005.06.042&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Schmidhuber%2C+J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-fernandez2007keyword-39">^ <a href="#cite_ref-fernandez2007keyword_39-0">a</a> <a href="#cite_ref-fernandez2007keyword_39-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFFernándezGravesSchmidhuber2007" class="citation conference cs1">Fernández, Santiago; Graves, Alex; Schmidhuber, Jürgen (2007). <a rel="nofollow" class="external text" href="http://dl.acm.org/citation.cfm?id=1778066.1778092">"An Application of Recurrent Neural Networks to Discriminative Keyword Spotting"</a>. Proceedings of the 17th International Conference on Artificial Neural Networks. ICANN'07. Berlin, Heidelberg: Springer-Verlag. pp. 220–229. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-74693-5" title="Special:BookSources/978-3-540-74693-5"><bdi>978-3-540-74693-5</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=An+Application+of+Recurrent+Neural+Networks+to+Discriminative+Keyword+Spotting&rft.btitle=Proceedings+of+the+17th+International+Conference+on+Artificial+Neural+Networks&rft.place=Berlin%2C+Heidelberg&rft.series=ICANN%2707&rft.pages=220-229&rft.pub=Springer-Verlag&rft.date=2007&rft.isbn=978-3-540-74693-5&rft.aulast=Fern%C3%A1ndez&rft.aufirst=Santiago&rft.au=Graves%2C+Alex&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fdl.acm.org%2Fcitation.cfm%3Fid%3D1778066.1778092&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-fan2015-40"><a href="#cite_ref-fan2015_40-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFFanWangSoongXie2015" class="citation conference cs1">Fan, Bo; Wang, Lijuan; Soong, Frank K.; Xie, Lei (2015). "Photo-Real Talking Head with Deep Bidirectional LSTM". Proceedings of ICASSP 2015 IEEE International Conference on Acoustics, Speech and Signal Processing. pp. 4884–8. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FICASSP.2015.7178899">10.1109/ICASSP.2015.7178899</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-4673-6997-8" title="Special:BookSources/978-1-4673-6997-8"><bdi>978-1-4673-6997-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Photo-Real+Talking+Head+with+Deep+Bidirectional+LSTM&rft.btitle=Proceedings+of+ICASSP+2015+IEEE+International+Conference+on+Acoustics%2C+Speech+and+Signal+Processing&rft.pages=4884-8&rft.date=2015&rft_id=info%3Adoi%2F10.1109%2FICASSP.2015.7178899&rft.isbn=978-1-4673-6997-8&rft.aulast=Fan&rft.aufirst=Bo&rft.au=Wang%2C+Lijuan&rft.au=Soong%2C+Frank+K.&rft.au=Xie%2C+Lei&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-sak2015-41"><a href="#cite_ref-sak2015_41-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSakSeniorRaoBeaufays2015" class="citation web cs1">Sak, Haşim; Senior, Andrew; Rao, Kanishka; Beaufays, Françoise; Schalkwyk, Johan (September 2015). <a rel="nofollow" class="external text" href="http://googleresearch.blogspot.ch/2015/09/google-voice-search-faster-and-more.html">"Google voice search: faster and more accurate"</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Google+voice+search%3A+faster+and+more+accurate&rft.date=2015-09&rft.aulast=Sak&rft.aufirst=Ha%C5%9Fim&rft.au=Senior%2C+Andrew&rft.au=Rao%2C+Kanishka&rft.au=Beaufays%2C+Fran%C3%A7oise&rft.au=Schalkwyk%2C+Johan&rft_id=http%3A%2F%2Fgoogleresearch.blogspot.ch%2F2015%2F09%2Fgoogle-voice-search-faster-and-more.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-sutskever2014-42">^ <a href="#cite_ref-sutskever2014_42-0">a</a> <a href="#cite_ref-sutskever2014_42-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSutskeverVinyalsLe2014" class="citation journal cs1">Sutskever, Ilya; Vinyals, Oriol; Le, Quoc V. (2014). <a rel="nofollow" class="external text" href="https://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf">"Sequence to Sequence Learning with Neural Networks"</a> (PDF). Electronic Proceedings of the Neural Information Processing Systems Conference. 27: 5346. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1409.3215">1409.3215</a>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2014arXiv1409.3215S">2014arXiv1409.3215S</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Electronic+Proceedings+of+the+Neural+Information+Processing+Systems+Conference&rft.atitle=Sequence+to+Sequence+Learning+with+Neural+Networks&rft.volume=27&rft.pages=5346&rft.date=2014&rft_id=info%3Aarxiv%2F1409.3215&rft_id=info%3Abibcode%2F2014arXiv1409.3215S&rft.aulast=Sutskever&rft.aufirst=Ilya&rft.au=Vinyals%2C+Oriol&rft.au=Le%2C+Quoc+V.&rft_id=https%3A%2F%2Fpapers.nips.cc%2Fpaper%2F5346-sequence-to-sequence-learning-with-neural-networks.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-vinyals2016-43"><a href="#cite_ref-vinyals2016_43-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJozefowiczVinyalsSchusterShazeer2016" class="citation arxiv cs1">Jozefowicz, Rafal; Vinyals, Oriol; Schuster, Mike; Shazeer, Noam; Wu, Yonghui (2016-02-07). "Exploring the Limits of Language Modeling". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1602.02410">1602.02410</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Exploring+the+Limits+of+Language+Modeling&rft.date=2016-02-07&rft_id=info%3Aarxiv%2F1602.02410&rft.aulast=Jozefowicz&rft.aufirst=Rafal&rft.au=Vinyals%2C+Oriol&rft.au=Schuster%2C+Mike&rft.au=Shazeer%2C+Noam&rft.au=Wu%2C+Yonghui&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-gillick2015-44"><a href="#cite_ref-gillick2015_44-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGillickBrunkVinyalsSubramanya2015" class="citation arxiv cs1">Gillick, Dan; Brunk, Cliff; Vinyals, Oriol; Subramanya, Amarnag (2015-11-30). "Multilingual Language Processing From Bytes". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1512.00103">1512.00103</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Multilingual+Language+Processing+From+Bytes&rft.date=2015-11-30&rft_id=info%3Aarxiv%2F1512.00103&rft.aulast=Gillick&rft.aufirst=Dan&rft.au=Brunk%2C+Cliff&rft.au=Vinyals%2C+Oriol&rft.au=Subramanya%2C+Amarnag&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-vinyals2015-45"><a href="#cite_ref-vinyals2015_45-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFVinyalsToshevBengioErhan2014" class="citation arxiv cs1">Vinyals, Oriol; Toshev, Alexander; Bengio, Samy; Erhan, Dumitru (2014-11-17). "Show and Tell: A Neural Image Caption Generator". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1411.4555">1411.4555</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CV">cs.CV</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Show+and+Tell%3A+A+Neural+Image+Caption+Generator&rft.date=2014-11-17&rft_id=info%3Aarxiv%2F1411.4555&rft.aulast=Vinyals&rft.aufirst=Oriol&rft.au=Toshev%2C+Alexander&rft.au=Bengio%2C+Samy&rft.au=Erhan%2C+Dumitru&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-:2-46"><a href="#cite_ref-:2_46-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChovan_MerrienboerGulcehreBahdanau2014" class="citation arxiv cs1">Cho, Kyunghyun; van Merrienboer, Bart; Gulcehre, Caglar; Bahdanau, Dzmitry; Bougares, Fethi; Schwenk, Holger; Bengio, Yoshua (2014-06-03). "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1406.1078">1406.1078</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Learning+Phrase+Representations+using+RNN+Encoder-Decoder+for+Statistical+Machine+Translation&rft.date=2014-06-03&rft_id=info%3Aarxiv%2F1406.1078&rft.aulast=Cho&rft.aufirst=Kyunghyun&rft.au=van+Merrienboer%2C+Bart&rft.au=Gulcehre%2C+Caglar&rft.au=Bahdanau%2C+Dzmitry&rft.au=Bougares%2C+Fethi&rft.au=Schwenk%2C+Holger&rft.au=Bengio%2C+Yoshua&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-sequence-47"><a href="#cite_ref-sequence_47-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSutskeverVinyalsLe2014" class="citation arxiv cs1">Sutskever, Ilya; Vinyals, Oriol; Le, Quoc Viet (14 Dec 2014). "Sequence to sequence learning with neural networks". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1409.3215">1409.3215</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Sequence+to+sequence+learning+with+neural+networks&rft.date=2014-12-14&rft_id=info%3Aarxiv%2F1409.3215&rft.aulast=Sutskever&rft.aufirst=Ilya&rft.au=Vinyals%2C+Oriol&rft.au=Le%2C+Quoc+Viet&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> [first version posted to arXiv on 10 Sep 2014] </li> <li id="cite_note-48"><a href="#cite_ref-48">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPetersNeumannIyyerGardner2018" class="citation arxiv cs1">Peters ME, Neumann M, Iyyer M, Gardner M, Clark C, Lee K, Zettlemoyer L (2018). "Deep contextualized word representations". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1802.05365">1802.05365</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Deep+contextualized+word+representations&rft.date=2018&rft_id=info%3Aarxiv%2F1802.05365&rft.aulast=Peters&rft.aufirst=ME&rft.au=Neumann%2C+M&rft.au=Iyyer%2C+M&rft.au=Gardner%2C+M&rft.au=Clark%2C+C&rft.au=Lee%2C+K&rft.au=Zettlemoyer%2C+L&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-49"><a href="#cite_ref-49">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFVaswaniShazeerParmarUszkoreit2017" class="citation journal cs1">Vaswani, Ashish; Shazeer, Noam; Parmar, Niki; Uszkoreit, Jakob; Jones, Llion; Gomez, Aidan N; Kaiser, Ł ukasz; Polosukhin, Illia (2017). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html">"Attention is All you Need"</a>. Advances in Neural Information Processing Systems. 30. Curran Associates, Inc.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Advances+in+Neural+Information+Processing+Systems&rft.atitle=Attention+is+All+you+Need&rft.volume=30&rft.date=2017&rft.aulast=Vaswani&rft.aufirst=Ashish&rft.au=Shazeer%2C+Noam&rft.au=Parmar%2C+Niki&rft.au=Uszkoreit%2C+Jakob&rft.au=Jones%2C+Llion&rft.au=Gomez%2C+Aidan+N&rft.au=Kaiser%2C+%C5%81+ukasz&rft.au=Polosukhin%2C+Illia&rft_id=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper%2F2017%2Fhash%2F3f5ee243547dee91fbd053c1c4a845aa-Abstract.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-50"><a href="#cite_ref-50">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFOordKalchbrennerKavukcuoglu2016" class="citation journal cs1">Oord, Aäron van den; Kalchbrenner, Nal; Kavukcuoglu, Koray (2016-06-11). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v48/oord16.html">"Pixel Recurrent Neural Networks"</a>. Proceedings of the 33rd International Conference on Machine Learning. PMLR: 1747–1756.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+the+33rd+International+Conference+on+Machine+Learning&rft.atitle=Pixel+Recurrent+Neural+Networks&rft.pages=1747-1756&rft.date=2016-06-11&rft.aulast=Oord&rft.aufirst=A%C3%A4ron+van+den&rft.au=Kalchbrenner%2C+Nal&rft.au=Kavukcuoglu%2C+Koray&rft_id=https%3A%2F%2Fproceedings.mlr.press%2Fv48%2Foord16.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-bmm615-51">^ <a href="#cite_ref-bmm615_51-0">a</a> <a href="#cite_ref-bmm615_51-1">b</a> Cruse, Holk; <a rel="nofollow" class="external text" href="http://www.brains-minds-media.org/archive/615/bmm615.pdf">Neural Networks as Cybernetic Systems</a>, 2nd and revised edition </li> <li id="cite_note-52"><a href="#cite_ref-52">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFElman1990" class="citation journal cs1">Elman, Jeffrey L. (1990). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0364-0213%2890%2990002-E">"Finding Structure in Time"</a>. Cognitive Science. 14 (2): 179–211. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0364-0213%2890%2990002-E">10.1016/0364-0213(90)90002-E</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Cognitive+Science&rft.atitle=Finding+Structure+in+Time&rft.volume=14&rft.issue=2&rft.pages=179-211&rft.date=1990&rft_id=info%3Adoi%2F10.1016%2F0364-0213%2890%2990002-E&rft.aulast=Elman&rft.aufirst=Jeffrey+L.&rft_id=https%3A%2F%2Fdoi.org%2F10.1016%252F0364-0213%252890%252990002-E&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-53"><a href="#cite_ref-53">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJordan1997" class="citation conference cs1">Jordan, Michael I. (1997-01-01). "Serial Order: A Parallel Distributed Processing Approach". Neural-Network Models of Cognition — Biobehavioral Foundations. Advances in Psychology. Vol. 121. pp. 471–495. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fs0166-4115%2897%2980111-2">10.1016/s0166-4115(97)80111-2</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-444-81931-4" title="Special:BookSources/978-0-444-81931-4"><bdi>978-0-444-81931-4</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15375627">15375627</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Serial+Order%3A+A+Parallel+Distributed+Processing+Approach&rft.btitle=Neural-Network+Models+of+Cognition+%E2%80%94+Biobehavioral+Foundations&rft.series=Advances+in+Psychology&rft.pages=471-495&rft.date=1997-01-01&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15375627%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1016%2Fs0166-4115%2897%2980111-2&rft.isbn=978-0-444-81931-4&rft.aulast=Jordan&rft.aufirst=Michael+I.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-gers2002-54"><a href="#cite_ref-gers2002_54-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGersSchraudolphSchmidhuber2002" class="citation journal cs1">Gers, Felix A.; Schraudolph, Nicol N.; Schmidhuber, Jürgen (2002). <a rel="nofollow" class="external text" href="http://www.jmlr.org/papers/volume3/gers02a/gers02a.pdf">"Learning Precise Timing with LSTM Recurrent Networks"</a> (PDF). Journal of Machine Learning Research. 3: 115–143. Retrieved 2017-06-13.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Machine+Learning+Research&rft.atitle=Learning+Precise+Timing+with+LSTM+Recurrent+Networks&rft.volume=3&rft.pages=115-143&rft.date=2002&rft.aulast=Gers&rft.aufirst=Felix+A.&rft.au=Schraudolph%2C+Nicol+N.&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fwww.jmlr.org%2Fpapers%2Fvolume3%2Fgers02a%2Fgers02a.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-hochreiter1991-55">^ <a href="#cite_ref-hochreiter1991_55-0">a</a> <a href="#cite_ref-hochreiter1991_55-1">b</a> <a href="#cite_ref-hochreiter1991_55-2">c</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHochreiter1991" class="citation thesis cs1">Hochreiter, Sepp (1991). <a rel="nofollow" class="external text" href="http://people.idsia.ch/~juergen/SeppHochreiter1991ThesisAdvisorSchmidhuber.pdf">Untersuchungen zu dynamischen neuronalen Netzen</a> (PDF) (Diploma). Institut f. Informatik, Technische University Munich.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Adissertation&rft.title=Untersuchungen+zu+dynamischen+neuronalen+Netzen&rft.inst=Institut+f.+Informatik%2C+Technische+University+Munich&rft.date=1991&rft.aulast=Hochreiter&rft.aufirst=Sepp&rft_id=http%3A%2F%2Fpeople.idsia.ch%2F~juergen%2FSeppHochreiter1991ThesisAdvisorSchmidhuber.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-bayer2009-56"><a href="#cite_ref-bayer2009_56-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBayerWierstraTogeliusSchmidhuber2009" class="citation book cs1">Bayer, Justin; Wierstra, Daan; Togelius, Julian; Schmidhuber, Jürgen (2009-09-14). "Evolving Memory Cell Structures for Sequence Learning". <a rel="nofollow" class="external text" href="https://mediatum.ub.tum.de/doc/1289041/document.pdf">Artificial Neural Networks – ICANN 2009</a> (PDF). Lecture Notes in Computer Science. Vol. 5769. Berlin, Heidelberg: Springer. pp. 755–764. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-642-04277-5_76">10.1007/978-3-642-04277-5_76</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-642-04276-8" title="Special:BookSources/978-3-642-04276-8"><bdi>978-3-642-04276-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Evolving+Memory+Cell+Structures+for+Sequence+Learning&rft.btitle=Artificial+Neural+Networks+%E2%80%93+ICANN+2009&rft.place=Berlin%2C+Heidelberg&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=755-764&rft.pub=Springer&rft.date=2009-09-14&rft_id=info%3Adoi%2F10.1007%2F978-3-642-04277-5_76&rft.isbn=978-3-642-04276-8&rft.aulast=Bayer&rft.aufirst=Justin&rft.au=Wierstra%2C+Daan&rft.au=Togelius%2C+Julian&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fmediatum.ub.tum.de%2Fdoc%2F1289041%2Fdocument.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-fernandez2007-57"><a href="#cite_ref-fernandez2007_57-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFFernándezGravesSchmidhuber2007" class="citation conference cs1">Fernández, Santiago; Graves, Alex; Schmidhuber, Jürgen (2007). <a rel="nofollow" class="external text" href="https://www.ijcai.org/Proceedings/07/Papers/124.pdf">"Sequence labelling in structured domains with hierarchical recurrent neural networks"</a> (PDF). Proceedings of the 20th International Joint Conference on Artificial Intelligence, Ijcai 2007. pp. 774–9. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.79.1887">10.1.1.79.1887</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Sequence+labelling+in+structured+domains+with+hierarchical+recurrent+neural+networks&rft.btitle=Proceedings+of+the+20th+International+Joint+Conference+on+Artificial+Intelligence%2C+Ijcai+2007&rft.pages=774-9&rft.date=2007&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.79.1887%23id-name%3DCiteSeerX&rft.aulast=Fern%C3%A1ndez&rft.aufirst=Santiago&rft.au=Graves%2C+Alex&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=https%3A%2F%2Fwww.ijcai.org%2FProceedings%2F07%2FPapers%2F124.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-peepholeLSTM-58">^ <a href="#cite_ref-peepholeLSTM_58-0">a</a> <a href="#cite_ref-peepholeLSTM_58-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGersSchmidhuber2001" class="citation journal cs1">Gers, Felix A.; Schmidhuber, Jürgen (2001). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20200710122825/ftp://ftp.idsia.ch/pub/juergen/L-IEEE.pdf">"LSTM Recurrent Networks Learn Simple Context Free and Context Sensitive Languages"</a> (PDF). IEEE Transactions on Neural Networks. 12 (6): 1333–40. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2F72.963769">10.1109/72.963769</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/18249962">18249962</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:10192330">10192330</a>. Archived from <a rel="nofollow" class="external text" href="ftp://ftp.idsia.ch/pub/juergen/L-IEEE.pdf">the original</a> (PDF) on 2020-07-10. Retrieved 2017-12-12.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Neural+Networks&rft.atitle=LSTM+Recurrent+Networks+Learn+Simple+Context+Free+and+Context+Sensitive+Languages&rft.volume=12&rft.issue=6&rft.pages=1333-40&rft.date=2001&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A10192330%23id-name%3DS2CID&rft_id=info%3Apmid%2F18249962&rft_id=info%3Adoi%2F10.1109%2F72.963769&rft.aulast=Gers&rft.aufirst=Felix+A.&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=ftp%3A%2F%2Fftp.idsia.ch%2Fpub%2Fjuergen%2FL-IEEE.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-59"><a href="#cite_ref-59">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHeckSalem2017" class="citation arxiv cs1">Heck, Joel; Salem, Fathi M. (2017-01-12). "Simplified Minimal Gated Unit Variations for Recurrent Neural Networks". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1701.03452">1701.03452</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Simplified+Minimal+Gated+Unit+Variations+for+Recurrent+Neural+Networks&rft.date=2017-01-12&rft_id=info%3Aarxiv%2F1701.03452&rft.aulast=Heck&rft.aufirst=Joel&rft.au=Salem%2C+Fathi+M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-60"><a href="#cite_ref-60">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDeySalem2017" class="citation arxiv cs1">Dey, Rahul; Salem, Fathi M. (2017-01-20). "Gate-Variants of Gated Recurrent Unit (GRU) Neural Networks". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1701.05923">1701.05923</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Gate-Variants+of+Gated+Recurrent+Unit+%28GRU%29+Neural+Networks&rft.date=2017-01-20&rft_id=info%3Aarxiv%2F1701.05923&rft.aulast=Dey&rft.aufirst=Rahul&rft.au=Salem%2C+Fathi+M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-MyUser_Wildml.com_May_18_2016c-61"><a href="#cite_ref-MyUser_Wildml.com_May_18_2016c_61-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBritz2015" class="citation web cs1">Britz, Denny (October 27, 2015). <a rel="nofollow" class="external text" href="http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/">"Recurrent Neural Network Tutorial, Part 4 – Implementing a GRU/LSTM RNN with Python and Theano – WildML"</a>. Wildml.com. Retrieved May 18, 2016.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=Wildml.com&rft.atitle=Recurrent+Neural+Network+Tutorial%2C+Part+4+%E2%80%93+Implementing+a+GRU%2FLSTM+RNN+with+Python+and+Theano+%E2%80%93+WildML&rft.date=2015-10-27&rft.aulast=Britz&rft.aufirst=Denny&rft_id=http%3A%2F%2Fwww.wildml.com%2F2015%2F10%2Frecurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-MyUser_Arxiv.org_May_18_2016c2-62">^ <a href="#cite_ref-MyUser_Arxiv.org_May_18_2016c2_62-0">a</a> <a href="#cite_ref-MyUser_Arxiv.org_May_18_2016c2_62-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChungGulcehreChoBengio2014" class="citation arxiv cs1">Chung, Junyoung; Gulcehre, Caglar; Cho, KyungHyun; Bengio, Yoshua (2014). "Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1412.3555">1412.3555</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Empirical+Evaluation+of+Gated+Recurrent+Neural+Networks+on+Sequence+Modeling&rft.date=2014&rft_id=info%3Aarxiv%2F1412.3555&rft.aulast=Chung&rft.aufirst=Junyoung&rft.au=Gulcehre%2C+Caglar&rft.au=Cho%2C+KyungHyun&rft.au=Bengio%2C+Yoshua&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-gruber_jockisch-63"><a href="#cite_ref-gruber_jockisch_63-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGruberJockisch2020" class="citation cs2">Gruber, N.; Jockisch, A. (2020), "Are GRU cells more specific and LSTM cells more sensitive in motive classification of text?", Frontiers in Artificial Intelligence, 3: 40, <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.3389%2Ffrai.2020.00040">10.3389/frai.2020.00040</a>, <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7861254">7861254</a>, <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/33733157">33733157</a>, <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:220252321">220252321</a></cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Frontiers+in+Artificial+Intelligence&rft.atitle=Are+GRU+cells+more+specific+and+LSTM+cells+more+sensitive+in+motive+classification+of+text%3F&rft.volume=3&rft.pages=40&rft.date=2020&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC7861254%23id-name%3DPMC&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A220252321%23id-name%3DS2CID&rft_id=info%3Apmid%2F33733157&rft_id=info%3Adoi%2F10.3389%2Ffrai.2020.00040&rft.aulast=Gruber&rft.aufirst=N.&rft.au=Jockisch%2C+A.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-64"><a href="#cite_ref-64">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKosko1988" class="citation journal cs1">Kosko, Bart (1988). "Bidirectional associative memories". IEEE Transactions on Systems, Man, and Cybernetics. 18 (1): 49–60. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2F21.87054">10.1109/21.87054</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:59875735">59875735</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Systems%2C+Man%2C+and+Cybernetics&rft.atitle=Bidirectional+associative+memories&rft.volume=18&rft.issue=1&rft.pages=49-60&rft.date=1988&rft_id=info%3Adoi%2F10.1109%2F21.87054&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A59875735%23id-name%3DS2CID&rft.aulast=Kosko&rft.aufirst=Bart&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-65"><a href="#cite_ref-65">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRakkiyappanChandrasekarLakshmananPark2015" class="citation journal cs1">Rakkiyappan, Rajan; Chandrasekar, Arunachalam; Lakshmanan, Subramanian; Park, Ju H. (2 January 2015). "Exponential stability for markovian jumping stochastic BAM neural networks with mode-dependent probabilistic time-varying delays and impulse control". Complexity. 20 (3): 39–65. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2015Cmplx..20c..39R">2015Cmplx..20c..39R</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fcplx.21503">10.1002/cplx.21503</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Complexity&rft.atitle=Exponential+stability+for+markovian+jumping+stochastic+BAM+neural+networks+with+mode-dependent+probabilistic+time-varying+delays+and+impulse+control&rft.volume=20&rft.issue=3&rft.pages=39-65&rft.date=2015-01-02&rft_id=info%3Adoi%2F10.1002%2Fcplx.21503&rft_id=info%3Abibcode%2F2015Cmplx..20c..39R&rft.aulast=Rakkiyappan&rft.aufirst=Rajan&rft.au=Chandrasekar%2C+Arunachalam&rft.au=Lakshmanan%2C+Subramanian&rft.au=Park%2C+Ju+H.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-66"><a href="#cite_ref-66">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRojas1996" class="citation book cs1">Rojas, Rául (1996). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=txsjjYzFJS4C&pg=PA336">Neural networks: a systematic introduction</a>. Springer. p. 336. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-60505-8" title="Special:BookSources/978-3-540-60505-8"><bdi>978-3-540-60505-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Neural+networks%3A+a+systematic+introduction&rft.pages=336&rft.pub=Springer&rft.date=1996&rft.isbn=978-3-540-60505-8&rft.aulast=Rojas&rft.aufirst=R%C3%A1ul&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3DtxsjjYzFJS4C%26pg%3DPA336&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-67"><a href="#cite_ref-67">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJaegerHaas2004" class="citation journal cs1">Jaeger, Herbert; Haas, Harald (2004-04-02). "Harnessing Nonlinearity: Predicting Chaotic Systems and Saving Energy in Wireless Communication". Science. 304 (5667): 78–80. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2004Sci...304...78J">2004Sci...304...78J</a>. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.719.2301">10.1.1.719.2301</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1126%2Fscience.1091277">10.1126/science.1091277</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/15064413">15064413</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:2184251">2184251</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Science&rft.atitle=Harnessing+Nonlinearity%3A+Predicting+Chaotic+Systems+and+Saving+Energy+in+Wireless+Communication&rft.volume=304&rft.issue=5667&rft.pages=78-80&rft.date=2004-04-02&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A2184251%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2004Sci...304...78J&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.719.2301%23id-name%3DCiteSeerX&rft_id=info%3Apmid%2F15064413&rft_id=info%3Adoi%2F10.1126%2Fscience.1091277&rft.aulast=Jaeger&rft.aufirst=Herbert&rft.au=Haas%2C+Harald&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-68"><a href="#cite_ref-68">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMaassNatschlägerMarkram2002" class="citation journal cs1">Maass, Wolfgang; Natschläger, Thomas; Markram, Henry (2002). <a rel="nofollow" class="external text" href="https://igi-web.tugraz.at/people/maass/psfiles/130.pdf">"Real-time computing without stable states: a new framework for neural computation based on perturbations"</a> (PDF). Neural Computation. 14 (11): 2531–2560. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2F089976602760407955">10.1162/089976602760407955</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/12433288">12433288</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1045112">1045112</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Real-time+computing+without+stable+states%3A+a+new+framework+for+neural+computation+based+on+perturbations&rft.volume=14&rft.issue=11&rft.pages=2531-2560&rft.date=2002&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1045112%23id-name%3DS2CID&rft_id=info%3Apmid%2F12433288&rft_id=info%3Adoi%2F10.1162%2F089976602760407955&rft.aulast=Maass&rft.aufirst=Wolfgang&rft.au=Natschl%C3%A4ger%2C+Thomas&rft.au=Markram%2C+Henry&rft_id=https%3A%2F%2Figi-web.tugraz.at%2Fpeople%2Fmaass%2Fpsfiles%2F130.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-69"><a href="#cite_ref-69">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGollerKüchler1996" class="citation book cs1">Goller, Christoph; Küchler, Andreas (1996). "Learning task-dependent distributed representations by backpropagation through structure". Proceedings of International Conference on Neural Networks (ICNN'96). Vol. 1. p. 347. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.52.4759">10.1.1.52.4759</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FICNN.1996.548916">10.1109/ICNN.1996.548916</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-7803-3210-2" title="Special:BookSources/978-0-7803-3210-2"><bdi>978-0-7803-3210-2</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:6536466">6536466</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Learning+task-dependent+distributed+representations+by+backpropagation+through+structure&rft.btitle=Proceedings+of+International+Conference+on+Neural+Networks+%28ICNN%2796%29&rft.pages=347&rft.date=1996&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.52.4759%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A6536466%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FICNN.1996.548916&rft.isbn=978-0-7803-3210-2&rft.aulast=Goller&rft.aufirst=Christoph&rft.au=K%C3%BCchler%2C+Andreas&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-lin1970-70"><a href="#cite_ref-lin1970_70-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLinnainmaa1970" class="citation thesis cs1 cs1-prop-foreign-lang-source"><a href="/wiki/Seppo_Linnainmaa" title="Seppo Linnainmaa">Linnainmaa, Seppo</a> (1970). The representation of the cumulative rounding error of an algorithm as a Taylor expansion of the local rounding errors (MSc) (in Finnish). University of Helsinki.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Adissertation&rft.title=The+representation+of+the+cumulative+rounding+error+of+an+algorithm+as+a+Taylor+expansion+of+the+local+rounding+errors&rft.inst=University+of+Helsinki&rft.date=1970&rft.aulast=Linnainmaa&rft.aufirst=Seppo&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-grie2008-71"><a href="#cite_ref-grie2008_71-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGriewankWalther2008" class="citation book cs1">Griewank, Andreas; <a href="/wiki/Andrea_Walther" title="Andrea Walther">Walther, Andrea</a> (2008). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=xoiiLaRxcbEC">Evaluating Derivatives: Principles and Techniques of Algorithmic Differentiation</a> (Second ed.). SIAM. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-89871-776-1" title="Special:BookSources/978-0-89871-776-1"><bdi>978-0-89871-776-1</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Evaluating+Derivatives%3A+Principles+and+Techniques+of+Algorithmic+Differentiation&rft.edition=Second&rft.pub=SIAM&rft.date=2008&rft.isbn=978-0-89871-776-1&rft.aulast=Griewank&rft.aufirst=Andreas&rft.au=Walther%2C+Andrea&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3DxoiiLaRxcbEC&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-72"><a href="#cite_ref-72">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSocherLinNgManning" class="citation cs2">Socher, Richard; Lin, Cliff; Ng, Andrew Y.; Manning, Christopher D., <a rel="nofollow" class="external text" href="https://ai.stanford.edu/~ang/papers/icml11-ParsingWithRecursiveNeuralNetworks.pdf">"Parsing Natural Scenes and Natural Language with Recursive Neural Networks"</a> (PDF), 28th International Conference on Machine Learning (ICML 2011)</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Parsing+Natural+Scenes+and+Natural+Language+with+Recursive+Neural+Networks&rft.btitle=28th+International+Conference+on+Machine+Learning+%28ICML+2011%29&rft.aulast=Socher&rft.aufirst=Richard&rft.au=Lin%2C+Cliff&rft.au=Ng%2C+Andrew+Y.&rft.au=Manning%2C+Christopher+D.&rft_id=https%3A%2F%2Fai.stanford.edu%2F~ang%2Fpapers%2Ficml11-ParsingWithRecursiveNeuralNetworks.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-73"><a href="#cite_ref-73">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSocherPerelyginWuChuang" class="citation journal cs1">Socher, Richard; Perelygin, Alex; Wu, Jean Y.; Chuang, Jason; Manning, Christopher D.; Ng, Andrew Y.; Potts, Christopher. <a rel="nofollow" class="external text" href="http://nlp.stanford.edu/~socherr/EMNLP2013_RNTN.pdf">"Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank"</a> (PDF). Emnlp 2013.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Emnlp+2013&rft.atitle=Recursive+Deep+Models+for+Semantic+Compositionality+Over+a+Sentiment+Treebank&rft.aulast=Socher&rft.aufirst=Richard&rft.au=Perelygin%2C+Alex&rft.au=Wu%2C+Jean+Y.&rft.au=Chuang%2C+Jason&rft.au=Manning%2C+Christopher+D.&rft.au=Ng%2C+Andrew+Y.&rft.au=Potts%2C+Christopher&rft_id=http%3A%2F%2Fnlp.stanford.edu%2F~socherr%2FEMNLP2013_RNTN.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-74"><a href="#cite_ref-74">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesWayneDanihelka2014" class="citation arxiv cs1">Graves, Alex; Wayne, Greg; Danihelka, Ivo (2014). "Neural Turing Machines". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1410.5401">1410.5401</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Neural+Turing+Machines&rft.date=2014&rft_id=info%3Aarxiv%2F1410.5401&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Wayne%2C+Greg&rft.au=Danihelka%2C+Ivo&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-DNCnature2016-75"><a href="#cite_ref-DNCnature2016_75-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesWayneReynoldsHarley2016" class="citation journal cs1">Graves, Alex; Wayne, Greg; Reynolds, Malcolm; Harley, Tim; Danihelka, Ivo; Grabska-Barwińska, Agnieszka; Colmenarejo, Sergio Gómez; Grefenstette, Edward; Ramalho, Tiago (2016-10-12). <a rel="nofollow" class="external text" href="http://www.nature.com/articles/nature20101.epdf?author_access_token=ImTXBI8aWbYxYQ51Plys8NRgN0jAjWel9jnR3ZoTv0MggmpDmwljGswxVdeocYSurJ3hxupzWuRNeGvvXnoO8o4jTJcnAyhGuZzXJ1GEaD-Z7E6X_a9R-xqJ9TfJWBqz">"Hybrid computing using a neural network with dynamic external memory"</a>. Nature. 538 (7626): 471–476. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2016Natur.538..471G">2016Natur.538..471G</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fnature20101">10.1038/nature20101</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1476-4687">1476-4687</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/27732574">27732574</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:205251479">205251479</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Nature&rft.atitle=Hybrid+computing+using+a+neural+network+with+dynamic+external+memory&rft.volume=538&rft.issue=7626&rft.pages=471-476&rft.date=2016-10-12&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A205251479%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2016Natur.538..471G&rft.issn=1476-4687&rft_id=info%3Adoi%2F10.1038%2Fnature20101&rft_id=info%3Apmid%2F27732574&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Wayne%2C+Greg&rft.au=Reynolds%2C+Malcolm&rft.au=Harley%2C+Tim&rft.au=Danihelka%2C+Ivo&rft.au=Grabska-Barwi%C5%84ska%2C+Agnieszka&rft.au=Colmenarejo%2C+Sergio+G%C3%B3mez&rft.au=Grefenstette%2C+Edward&rft.au=Ramalho%2C+Tiago&rft_id=http%3A%2F%2Fwww.nature.com%2Farticles%2Fnature20101.epdf%3Fauthor_access_token%3DImTXBI8aWbYxYQ51Plys8NRgN0jAjWel9jnR3ZoTv0MggmpDmwljGswxVdeocYSurJ3hxupzWuRNeGvvXnoO8o4jTJcnAyhGuZzXJ1GEaD-Z7E6X_a9R-xqJ9TfJWBqz&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-76"><a href="#cite_ref-76">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSunGilesChen1998" class="citation book cs1">Sun, Guo-Zheng; Giles, C. Lee; Chen, Hsing-Hen (1998). "The Neural Network Pushdown Automaton: Architecture, Dynamics and Training". In Giles, C. Lee; Gori, Marco (eds.). Adaptive Processing of Sequences and Data Structures. Lecture Notes in Computer Science. Berlin, Heidelberg: Springer. pp. 296–345. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.56.8723">10.1.1.56.8723</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fbfb0054003">10.1007/bfb0054003</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-64341-8" title="Special:BookSources/978-3-540-64341-8"><bdi>978-3-540-64341-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=The+Neural+Network+Pushdown+Automaton%3A+Architecture%2C+Dynamics+and+Training&rft.btitle=Adaptive+Processing+of+Sequences+and+Data+Structures&rft.place=Berlin%2C+Heidelberg&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=296-345&rft.pub=Springer&rft.date=1998&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.56.8723%23id-name%3DCiteSeerX&rft_id=info%3Adoi%2F10.1007%2Fbfb0054003&rft.isbn=978-3-540-64341-8&rft.aulast=Sun&rft.aufirst=Guo-Zheng&rft.au=Giles%2C+C.+Lee&rft.au=Chen%2C+Hsing-Hen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-77"><a href="#cite_ref-77">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHyötyniemi1996" class="citation journal cs1">Hyötyniemi, Heikki (1996). "Turing machines are recurrent neural networks". Proceedings of STeP '96/Publications of the Finnish Artificial Intelligence Society: 13–24.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+STeP+%2796%2FPublications+of+the+Finnish+Artificial+Intelligence+Society&rft.atitle=Turing+machines+are+recurrent+neural+networks&rft.pages=13-24&rft.date=1996&rft.aulast=Hy%C3%B6tyniemi&rft.aufirst=Heikki&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-78"><a href="#cite_ref-78">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRobinsonFallside1987" class="citation book cs1">Robinson, Anthony J.; Fallside, Frank (1987). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=6JYYMwEACAAJ">The Utility Driven Dynamic Error Propagation Network</a>. Technical Report CUED/F-INFENG/TR.1. Department of Engineering, University of Cambridge.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=The+Utility+Driven+Dynamic+Error+Propagation+Network&rft.series=Technical+Report+CUED%2FF-INFENG%2FTR.1&rft.pub=Department+of+Engineering%2C+University+of+Cambridge&rft.date=1987&rft.aulast=Robinson&rft.aufirst=Anthony+J.&rft.au=Fallside%2C+Frank&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3D6JYYMwEACAAJ&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-79"><a href="#cite_ref-79">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWilliamsZipser2013" class="citation book cs1">Williams, Ronald J.; Zipser, D. (1 February 2013). "Gradient-based learning algorithms for recurrent networks and their computational complexity". In Chauvin, Yves; Rumelhart, David E. (eds.). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=B71nu3LDpREC">Backpropagation: Theory, Architectures, and Applications</a>. Psychology Press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-134-77581-1" title="Special:BookSources/978-1-134-77581-1"><bdi>978-1-134-77581-1</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Gradient-based+learning+algorithms+for+recurrent+networks+and+their+computational+complexity&rft.btitle=Backpropagation%3A+Theory%2C+Architectures%2C+and+Applications&rft.pub=Psychology+Press&rft.date=2013-02-01&rft.isbn=978-1-134-77581-1&rft.aulast=Williams&rft.aufirst=Ronald+J.&rft.au=Zipser%2C+D.&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3DB71nu3LDpREC&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-80"><a href="#cite_ref-80">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSchmidhuber1989" class="citation journal cs1">Schmidhuber, Jürgen (1989-01-01). "A Local Learning Algorithm for Dynamic Feedforward and Recurrent Networks". Connection Science. 1 (4): 403–412. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F09540098908915650">10.1080/09540098908915650</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:18721007">18721007</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Connection+Science&rft.atitle=A+Local+Learning+Algorithm+for+Dynamic+Feedforward+and+Recurrent+Networks&rft.volume=1&rft.issue=4&rft.pages=403-412&rft.date=1989-01-01&rft_id=info%3Adoi%2F10.1080%2F09540098908915650&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A18721007%23id-name%3DS2CID&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-PríncipeEuliano2000-81"><a href="#cite_ref-PríncipeEuliano2000_81-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPríncipeEulianoLefebvre2000" class="citation book cs1">Príncipe, José C.; Euliano, Neil R.; Lefebvre, W. Curt (2000). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=jgMZAQAAIAAJ">Neural and adaptive systems: fundamentals through simulations</a>. Wiley. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-471-35167-2" title="Special:BookSources/978-0-471-35167-2"><bdi>978-0-471-35167-2</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Neural+and+adaptive+systems%3A+fundamentals+through+simulations&rft.pub=Wiley&rft.date=2000&rft.isbn=978-0-471-35167-2&rft.aulast=Pr%C3%ADncipe&rft.aufirst=Jos%C3%A9+C.&rft.au=Euliano%2C+Neil+R.&rft.au=Lefebvre%2C+W.+Curt&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3DjgMZAQAAIAAJ&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Ollivier2015-82"><a href="#cite_ref-Ollivier2015_82-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFYannTallecCharpiat2015" class="citation arxiv cs1">Yann, Ollivier; Tallec, Corentin; Charpiat, Guillaume (2015-07-28). "Training recurrent networks online without backtracking". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1507.07680">1507.07680</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Training+recurrent+networks+online+without+backtracking&rft.date=2015-07-28&rft_id=info%3Aarxiv%2F1507.07680&rft.aulast=Yann&rft.aufirst=Ollivier&rft.au=Tallec%2C+Corentin&rft.au=Charpiat%2C+Guillaume&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-83"><a href="#cite_ref-83">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSchmidhuber1992" class="citation journal cs1">Schmidhuber, Jürgen (1992-03-01). "A Fixed Size Storage O(n3) Time Complexity Learning Algorithm for Fully Recurrent Continually Running Networks". Neural Computation. 4 (2): 243–248. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fneco.1992.4.2.243">10.1162/neco.1992.4.2.243</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:11761172">11761172</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=A+Fixed+Size+Storage+O%28n3%29+Time+Complexity+Learning+Algorithm+for+Fully+Recurrent+Continually+Running+Networks&rft.volume=4&rft.issue=2&rft.pages=243-248&rft.date=1992-03-01&rft_id=info%3Adoi%2F10.1162%2Fneco.1992.4.2.243&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A11761172%23id-name%3DS2CID&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-84"><a href="#cite_ref-84">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWilliams1989" class="citation report cs1">Williams, Ronald J. (1989). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20171020033840/http://citeseerx.ist.psu.edu/showciting?cid=128036">Complexity of exact gradient computation algorithms for recurrent neural networks</a> (Report). Technical Report NU-CCS-89-27. Boston (MA): Northeastern University, College of Computer Science. Archived from <a rel="nofollow" class="external text" href="http://citeseerx.ist.psu.edu/showciting?cid=128036">the original</a> on 2017-10-20. Retrieved 2017-07-02.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=report&rft.btitle=Complexity+of+exact+gradient+computation+algorithms+for+recurrent+neural+networks&rft.place=Boston+%28MA%29&rft.series=Technical+Report+NU-CCS-89-27&rft.pub=Northeastern+University%2C+College+of+Computer+Science&rft.date=1989&rft.aulast=Williams&rft.aufirst=Ronald+J.&rft_id=http%3A%2F%2Fciteseerx.ist.psu.edu%2Fshowciting%3Fcid%3D128036&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-85"><a href="#cite_ref-85">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPearlmutter1989" class="citation journal cs1">Pearlmutter, Barak A. (1989-06-01). <a rel="nofollow" class="external text" href="http://repository.cmu.edu/cgi/viewcontent.cgi?article=2865&context=compsci">"Learning State Space Trajectories in Recurrent Neural Networks"</a>. Neural Computation. 1 (2): 263–269. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fneco.1989.1.2.263">10.1162/neco.1989.1.2.263</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:16813485">16813485</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Learning+State+Space+Trajectories+in+Recurrent+Neural+Networks&rft.volume=1&rft.issue=2&rft.pages=263-269&rft.date=1989-06-01&rft_id=info%3Adoi%2F10.1162%2Fneco.1989.1.2.263&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A16813485%23id-name%3DS2CID&rft.aulast=Pearlmutter&rft.aufirst=Barak+A.&rft_id=http%3A%2F%2Frepository.cmu.edu%2Fcgi%2Fviewcontent.cgi%3Farticle%3D2865%26context%3Dcompsci&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-HOCH2001-86"><a href="#cite_ref-HOCH2001_86-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHochreiter2001" class="citation book cs1">Hochreiter, Sepp; et al. (15 January 2001). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=NWOcMVA64aAC">"Gradient flow in recurrent nets: the difficulty of learning long-term dependencies"</a>. In Kolen, John F.; Kremer, Stefan C. (eds.). A Field Guide to Dynamical Recurrent Networks. John Wiley & Sons. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-7803-5369-5" title="Special:BookSources/978-0-7803-5369-5"><bdi>978-0-7803-5369-5</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Gradient+flow+in+recurrent+nets%3A+the+difficulty+of+learning+long-term+dependencies&rft.btitle=A+Field+Guide+to+Dynamical+Recurrent+Networks&rft.pub=John+Wiley+%26+Sons&rft.date=2001-01-15&rft.isbn=978-0-7803-5369-5&rft.aulast=Hochreiter&rft.aufirst=Sepp&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3DNWOcMVA64aAC&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-auto-87">^ <a href="#cite_ref-auto_87-0">a</a> <a href="#cite_ref-auto_87-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLiLiCookZhu2018" class="citation arxiv cs1">Li, Shuai; Li, Wanqing; Cook, Chris; Zhu, Ce; Yanbo, Gao (2018). "Independently Recurrent Neural Network (IndRNN): Building a Longer and Deeper RNN". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1803.04831">1803.04831</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CV">cs.CV</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Independently+Recurrent+Neural+Network+%28IndRNN%29%3A+Building+a+Longer+and+Deeper+RNN&rft.date=2018&rft_id=info%3Aarxiv%2F1803.04831&rft.aulast=Li&rft.aufirst=Shuai&rft.au=Li%2C+Wanqing&rft.au=Cook%2C+Chris&rft.au=Zhu%2C+Ce&rft.au=Yanbo%2C+Gao&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-88"><a href="#cite_ref-88">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCampolucciUnciniPiazzaRao1999" class="citation journal cs1">Campolucci, Paolo; Uncini, Aurelio; Piazza, Francesco; Rao, Bhaskar D. (1999). "On-Line Learning Algorithms for Locally Recurrent Neural Networks". IEEE Transactions on Neural Networks. 10 (2): 253–271. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.33.7550">10.1.1.33.7550</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2F72.750549">10.1109/72.750549</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/18252525">18252525</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Neural+Networks&rft.atitle=On-Line+Learning+Algorithms+for+Locally+Recurrent+Neural+Networks&rft.volume=10&rft.issue=2&rft.pages=253-271&rft.date=1999&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.33.7550%23id-name%3DCiteSeerX&rft_id=info%3Apmid%2F18252525&rft_id=info%3Adoi%2F10.1109%2F72.750549&rft.aulast=Campolucci&rft.aufirst=Paolo&rft.au=Uncini%2C+Aurelio&rft.au=Piazza%2C+Francesco&rft.au=Rao%2C+Bhaskar+D.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-89"><a href="#cite_ref-89">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWanBeaufays1996" class="citation journal cs1">Wan, Eric A.; Beaufays, Françoise (1996). "Diagrammatic derivation of gradient algorithms for neural networks". Neural Computation. 8: 182–201. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fneco.1996.8.1.182">10.1162/neco.1996.8.1.182</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15512077">15512077</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Diagrammatic+derivation+of+gradient+algorithms+for+neural+networks&rft.volume=8&rft.pages=182-201&rft.date=1996&rft_id=info%3Adoi%2F10.1162%2Fneco.1996.8.1.182&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15512077%23id-name%3DS2CID&rft.aulast=Wan&rft.aufirst=Eric+A.&rft.au=Beaufays%2C+Fran%C3%A7oise&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-ReferenceA-90">^ <a href="#cite_ref-ReferenceA_90-0">a</a> <a href="#cite_ref-ReferenceA_90-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCampolucciUnciniPiazza2000" class="citation journal cs1">Campolucci, Paolo; Uncini, Aurelio; Piazza, Francesco (2000). "A Signal-Flow-Graph Approach to On-line Gradient Calculation". Neural Computation. 12 (8): 1901–1927. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.212.5406">10.1.1.212.5406</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2F089976600300015196">10.1162/089976600300015196</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/10953244">10953244</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15090951">15090951</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=A+Signal-Flow-Graph+Approach+to+On-line+Gradient+Calculation&rft.volume=12&rft.issue=8&rft.pages=1901-1927&rft.date=2000&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.212.5406%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15090951%23id-name%3DS2CID&rft_id=info%3Apmid%2F10953244&rft_id=info%3Adoi%2F10.1162%2F089976600300015196&rft.aulast=Campolucci&rft.aufirst=Paolo&rft.au=Uncini%2C+Aurelio&rft.au=Piazza%2C+Francesco&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-graves2006-91"><a href="#cite_ref-graves2006_91-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesFernándezGomez2006" class="citation conference cs1">Graves, Alex; Fernández, Santiago; Gomez, Faustino J. (2006). <a rel="nofollow" class="external text" href="https://axon.cs.byu.edu/~martinez/classes/778/Papers/p369-graves.pdf">"Connectionist temporal classification: Labelling unsegmented sequence data with recurrent neural networks"</a> (PDF). Proceedings of the International Conference on Machine Learning. pp. 369–376. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.75.6306">10.1.1.75.6306</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F1143844.1143891">10.1145/1143844.1143891</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/1-59593-383-2" title="Special:BookSources/1-59593-383-2"><bdi>1-59593-383-2</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Connectionist+temporal+classification%3A+Labelling+unsegmented+sequence+data+with+recurrent+neural+networks&rft.btitle=Proceedings+of+the+International+Conference+on+Machine+Learning&rft.pages=369-376&rft.date=2006&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.75.6306%23id-name%3DCiteSeerX&rft_id=info%3Adoi%2F10.1145%2F1143844.1143891&rft.isbn=1-59593-383-2&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Fern%C3%A1ndez%2C+Santiago&rft.au=Gomez%2C+Faustino+J.&rft_id=https%3A%2F%2Faxon.cs.byu.edu%2F~martinez%2Fclasses%2F778%2FPapers%2Fp369-graves.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-92"><a href="#cite_ref-92">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHannun2017" class="citation journal cs1">Hannun, Awni (2017-11-27). <a rel="nofollow" class="external text" href="https://distill.pub/2017/ctc">"Sequence Modeling with CTC"</a>. Distill. 2 (11): e8. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.23915%2Fdistill.00008">10.23915/distill.00008</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2476-0757">2476-0757</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Distill&rft.atitle=Sequence+Modeling+with+CTC&rft.volume=2&rft.issue=11&rft.pages=e8&rft.date=2017-11-27&rft_id=info%3Adoi%2F10.23915%2Fdistill.00008&rft.issn=2476-0757&rft.aulast=Hannun&rft.aufirst=Awni&rft_id=https%3A%2F%2Fdistill.pub%2F2017%2Fctc&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-93"><a href="#cite_ref-93">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGomezMiikkulainen1999" class="citation cs2">Gomez, Faustino J.; Miikkulainen, Risto (1999), <a rel="nofollow" class="external text" href="http://www.cs.utexas.edu/users/nn/downloads/papers/gomez.ijcai99.pdf">"Solving non-Markovian control tasks with neuroevolution"</a> (PDF), IJCAI 99, Morgan Kaufmann, retrieved 5 August 2017</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Solving+non-Markovian+control+tasks+with+neuroevolution&rft.btitle=IJCAI+99&rft.pub=Morgan+Kaufmann&rft.date=1999&rft.aulast=Gomez&rft.aufirst=Faustino+J.&rft.au=Miikkulainen%2C+Risto&rft_id=http%3A%2F%2Fwww.cs.utexas.edu%2Fusers%2Fnn%2Fdownloads%2Fpapers%2Fgomez.ijcai99.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-94"><a href="#cite_ref-94">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSyed1995" class="citation thesis cs1">Syed, Omar (May 1995). <a rel="nofollow" class="external text" href="http://arimaa.com/arimaa/about/Thesis/">Applying Genetic Algorithms to Recurrent Neural Networks for Learning Network Parameters and Architecture</a> (MSc). Department of Electrical Engineering, Case Western Reserve University.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Adissertation&rft.title=Applying+Genetic+Algorithms+to+Recurrent+Neural+Networks+for+Learning+Network+Parameters+and+Architecture&rft.inst=Department+of+Electrical+Engineering%2C+Case+Western+Reserve+University&rft.date=1995-05&rft.aulast=Syed&rft.aufirst=Omar&rft_id=http%3A%2F%2Farimaa.com%2Farimaa%2Fabout%2FThesis%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-95"><a href="#cite_ref-95">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGomezSchmidhuberMiikkulainen2008" class="citation journal cs1">Gomez, Faustino J.; Schmidhuber, Jürgen; Miikkulainen, Risto (June 2008). <a rel="nofollow" class="external text" href="https://www.jmlr.org/papers/volume9/gomez08a/gomez08a.pdf">"Accelerated Neural Evolution Through Cooperatively Coevolved Synapses"</a> (PDF). Journal of Machine Learning Research. 9: 937–965.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Machine+Learning+Research&rft.atitle=Accelerated+Neural+Evolution+Through+Cooperatively+Coevolved+Synapses&rft.volume=9&rft.pages=937-965&rft.date=2008-06&rft.aulast=Gomez&rft.aufirst=Faustino+J.&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft.au=Miikkulainen%2C+Risto&rft_id=https%3A%2F%2Fwww.jmlr.org%2Fpapers%2Fvolume9%2Fgomez08a%2Fgomez08a.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-schmidhuber1992-96">^ <a href="#cite_ref-schmidhuber1992_96-0">a</a> <a href="#cite_ref-schmidhuber1992_96-1">b</a> <a href="#cite_ref-schmidhuber1992_96-2">c</a> <a href="#cite_ref-schmidhuber1992_96-3">d</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSchmidhuber1992" class="citation journal cs1">Schmidhuber, Jürgen (1992). <a rel="nofollow" class="external text" href="ftp://ftp.idsia.ch/pub/juergen/chunker.pdf">"Learning complex, extended sequences using the principle of history compression"</a> (PDF). Neural Computation. 4 (2): 234–242. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fneco.1992.4.2.234">10.1162/neco.1992.4.2.234</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:18271205">18271205</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Learning+complex%2C+extended+sequences+using+the+principle+of+history+compression&rft.volume=4&rft.issue=2&rft.pages=234-242&rft.date=1992&rft_id=info%3Adoi%2F10.1162%2Fneco.1992.4.2.234&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A18271205%23id-name%3DS2CID&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rft_id=ftp%3A%2F%2Fftp.idsia.ch%2Fpub%2Fjuergen%2Fchunker.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988">[<a href="/wiki/Wikipedia:Link_rot" title="Wikipedia:Link rot">permanent dead link</a>‍] </li> <li id="cite_note-scholarpedia2015pre-97"><a href="#cite_ref-scholarpedia2015pre_97-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSchmidhuber2015" class="citation journal cs1">Schmidhuber, Jürgen (2015). <a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.32832">"Deep Learning"</a>. Scholarpedia. 10 (11): 32832. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2015SchpJ..1032832S">2015SchpJ..1032832S</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.32832">10.4249/scholarpedia.32832</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Scholarpedia&rft.atitle=Deep+Learning&rft.volume=10&rft.issue=11&rft.pages=32832&rft.date=2015&rft_id=info%3Adoi%2F10.4249%2Fscholarpedia.32832&rft_id=info%3Abibcode%2F2015SchpJ..1032832S&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rft_id=https%3A%2F%2Fdoi.org%2F10.4249%252Fscholarpedia.32832&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-98"><a href="#cite_ref-98">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGilesMillerChenChen1992" class="citation journal cs1">Giles, C. Lee; Miller, Clifford B.; Chen, Dong; Chen, Hsing-Hen; Sun, Guo-Zheng; Lee, Yee-Chun (1992). <a rel="nofollow" class="external text" href="https://clgiles.ist.psu.edu/pubs/NC1992-recurrent-NN.pdf">"Learning and Extracting Finite State Automata with Second-Order Recurrent Neural Networks"</a> (PDF). Neural Computation. 4 (3): 393–405. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2Fneco.1992.4.3.393">10.1162/neco.1992.4.3.393</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:19666035">19666035</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Learning+and+Extracting+Finite+State+Automata+with+Second-Order+Recurrent+Neural+Networks&rft.volume=4&rft.issue=3&rft.pages=393-405&rft.date=1992&rft_id=info%3Adoi%2F10.1162%2Fneco.1992.4.3.393&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A19666035%23id-name%3DS2CID&rft.aulast=Giles&rft.aufirst=C.+Lee&rft.au=Miller%2C+Clifford+B.&rft.au=Chen%2C+Dong&rft.au=Chen%2C+Hsing-Hen&rft.au=Sun%2C+Guo-Zheng&rft.au=Lee%2C+Yee-Chun&rft_id=https%3A%2F%2Fclgiles.ist.psu.edu%2Fpubs%2FNC1992-recurrent-NN.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-99"><a href="#cite_ref-99">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFOmlinGiles1996" class="citation journal cs1">Omlin, Christian W.; Giles, C. Lee (1996). "Constructing Deterministic Finite-State Automata in Recurrent Neural Networks". Journal of the ACM. 45 (6): 937–972. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.32.2364">10.1.1.32.2364</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F235809.235811">10.1145/235809.235811</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:228941">228941</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+ACM&rft.atitle=Constructing+Deterministic+Finite-State+Automata+in+Recurrent+Neural+Networks&rft.volume=45&rft.issue=6&rft.pages=937-972&rft.date=1996&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.32.2364%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A228941%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F235809.235811&rft.aulast=Omlin&rft.aufirst=Christian+W.&rft.au=Giles%2C+C.+Lee&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-100"><a href="#cite_ref-100">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPaineTani2005" class="citation journal cs1">Paine, Rainer W.; Tani, Jun (2005-09-01). "How Hierarchical Control Self-organizes in Artificial Adaptive Systems". Adaptive Behavior. 13 (3): 211–225. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F105971230501300303">10.1177/105971230501300303</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:9932565">9932565</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Adaptive+Behavior&rft.atitle=How+Hierarchical+Control+Self-organizes+in+Artificial+Adaptive+Systems&rft.volume=13&rft.issue=3&rft.pages=211-225&rft.date=2005-09-01&rft_id=info%3Adoi%2F10.1177%2F105971230501300303&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A9932565%23id-name%3DS2CID&rft.aulast=Paine&rft.aufirst=Rainer+W.&rft.au=Tani%2C+Jun&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-auto1-101">^ <a href="#cite_ref-auto1_101-0">a</a> <a href="#cite_ref-auto1_101-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/328474302">"Burns, Benureau, Tani (2018) A Bergson-Inspired Adaptive Time Constant for the Multiple Timescales Recurrent Neural Network Model. JNNS"</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Burns%2C+Benureau%2C+Tani+%282018%29+A+Bergson-Inspired+Adaptive+Time+Constant+for+the+Multiple+Timescales+Recurrent+Neural+Network+Model.+JNNS&rft_id=https%3A%2F%2Fwww.researchgate.net%2Fpublication%2F328474302&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-barkan-102"><a href="#cite_ref-barkan_102-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBarkanBenchimolCaspiCohen2023" class="citation journal cs1">Barkan, Oren; Benchimol, Jonathan; Caspi, Itamar; Cohen, Eliya; Hammer, Allon; Koenigstein, Noam (2023). "Forecasting CPI inflation components with Hierarchical Recurrent Neural Networks". International Journal of Forecasting. 39 (3): 1145–1162. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2011.07920">2011.07920</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ijforecast.2022.04.009">10.1016/j.ijforecast.2022.04.009</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=International+Journal+of+Forecasting&rft.atitle=Forecasting+CPI+inflation+components+with+Hierarchical+Recurrent+Neural+Networks&rft.volume=39&rft.issue=3&rft.pages=1145-1162&rft.date=2023&rft_id=info%3Aarxiv%2F2011.07920&rft_id=info%3Adoi%2F10.1016%2Fj.ijforecast.2022.04.009&rft.aulast=Barkan&rft.aufirst=Oren&rft.au=Benchimol%2C+Jonathan&rft.au=Caspi%2C+Itamar&rft.au=Cohen%2C+Eliya&rft.au=Hammer%2C+Allon&rft.au=Koenigstein%2C+Noam&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-103"><a href="#cite_ref-103">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFTutschku1995" class="citation book cs1">Tutschku, Kurt (June 1995). Recurrent Multilayer Perceptrons for Identification and Control: The Road to Applications. Institute of Computer Science Research Report. Vol. 118. University of Würzburg Am Hubland. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.45.3527">10.1.1.45.3527</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Recurrent+Multilayer+Perceptrons+for+Identification+and+Control%3A+The+Road+to+Applications&rft.series=Institute+of+Computer+Science+Research+Report&rft.pub=University+of+W%C3%BCrzburg+Am+Hubland&rft.date=1995-06&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.45.3527%23id-name%3DCiteSeerX&rft.aulast=Tutschku&rft.aufirst=Kurt&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-104"><a href="#cite_ref-104">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFYamashitaTani2008" class="citation journal cs1">Yamashita, Yuichi; Tani, Jun (2008-11-07). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2570613">"Emergence of Functional Hierarchy in a Multiple Timescale Neural Network Model: A Humanoid Robot Experiment"</a>. PLOS Computational Biology. 4 (11): e1000220. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2008PLSCB...4E0220Y">2008PLSCB...4E0220Y</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1371%2Fjournal.pcbi.1000220">10.1371/journal.pcbi.1000220</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2570613">2570613</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/18989398">18989398</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=PLOS+Computational+Biology&rft.atitle=Emergence+of+Functional+Hierarchy+in+a+Multiple+Timescale+Neural+Network+Model%3A+A+Humanoid+Robot+Experiment&rft.volume=4&rft.issue=11&rft.pages=e1000220&rft.date=2008-11-07&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC2570613%23id-name%3DPMC&rft_id=info%3Apmid%2F18989398&rft_id=info%3Adoi%2F10.1371%2Fjournal.pcbi.1000220&rft_id=info%3Abibcode%2F2008PLSCB...4E0220Y&rft.aulast=Yamashita&rft.aufirst=Yuichi&rft.au=Tani%2C+Jun&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC2570613&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-105"><a href="#cite_ref-105">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFAlnajjarYamashitaTani2013" class="citation journal cs1">Alnajjar, Fady; Yamashita, Yuichi; Tani, Jun (2013). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3575058">"The hierarchical and functional connectivity of higher-order cognitive mechanisms: neurorobotic model to investigate the stability and flexibility of working memory"</a>. Frontiers in Neurorobotics. 7: 2. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.3389%2Ffnbot.2013.00002">10.3389/fnbot.2013.00002</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3575058">3575058</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/23423881">23423881</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Frontiers+in+Neurorobotics&rft.atitle=The+hierarchical+and+functional+connectivity+of+higher-order+cognitive+mechanisms%3A+neurorobotic+model+to+investigate+the+stability+and+flexibility+of+working+memory&rft.volume=7&rft.pages=2&rft.date=2013&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC3575058%23id-name%3DPMC&rft_id=info%3Apmid%2F23423881&rft_id=info%3Adoi%2F10.3389%2Ffnbot.2013.00002&rft.aulast=Alnajjar&rft.aufirst=Fady&rft.au=Yamashita%2C+Yuichi&rft.au=Tani%2C+Jun&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC3575058&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-106"><a href="#cite_ref-106">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="http://jnns.org/conference/2018/JNNS2018_Technical_Programs.pdf">"Proceedings of the 28th Annual Conference of the Japanese Neural Network Society (October, 2018)"</a> (PDF).</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Proceedings+of+the+28th+Annual+Conference+of+the+Japanese+Neural+Network+Society+%28October%2C+2018%29&rft_id=http%3A%2F%2Fjnns.org%2Fconference%2F2018%2FJNNS2018_Technical_Programs.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-107"><a href="#cite_ref-107">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSnider2008" class="citation cs2">Snider, Greg (2008), <a rel="nofollow" class="external text" href="https://web.archive.org/web/20160516070906/http://www.scidacreview.org/0804/html/hardware.html">"Cortical computing with memristive nanodevices"</a>, Sci-DAC Review, 10: 58–65, archived from <a rel="nofollow" class="external text" href="http://www.scidacreview.org/0804/html/hardware.html">the original</a> on 2016-05-16, retrieved 2019-09-06</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Sci-DAC+Review&rft.atitle=Cortical+computing+with+memristive+nanodevices&rft.volume=10&rft.pages=58-65&rft.date=2008&rft.aulast=Snider&rft.aufirst=Greg&rft_id=http%3A%2F%2Fwww.scidacreview.org%2F0804%2Fhtml%2Fhardware.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-108"><a href="#cite_ref-108">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCaravelliTraversaDi_Ventra2017" class="citation journal cs1">Caravelli, Francesco; Traversa, Fabio Lorenzo; Di Ventra, Massimiliano (2017). "The complex dynamics of memristive circuits: analytical results and universal slow relaxation". Physical Review E. 95 (2): 022140. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1608.08651">1608.08651</a>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2017PhRvE..95b2140C">2017PhRvE..95b2140C</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevE.95.022140">10.1103/PhysRevE.95.022140</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/28297937">28297937</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:6758362">6758362</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+E&rft.atitle=The+complex+dynamics+of+memristive+circuits%3A+analytical+results+and+universal+slow+relaxation&rft.volume=95&rft.issue=2&rft.pages=022140&rft.date=2017&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A6758362%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2017PhRvE..95b2140C&rft_id=info%3Aarxiv%2F1608.08651&rft_id=info%3Apmid%2F28297937&rft_id=info%3Adoi%2F10.1103%2FPhysRevE.95.022140&rft.aulast=Caravelli&rft.aufirst=Francesco&rft.au=Traversa%2C+Fabio+Lorenzo&rft.au=Di+Ventra%2C+Massimiliano&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-109"><a href="#cite_ref-109">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHarveyHusbandsCliff1994" class="citation cs2">Harvey, Inman; Husbands, Phil; Cliff, Dave (1994), <a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/229091538_Seeing_the_Light_Artificial_Evolution_Real_Vision">"Seeing the light: Artificial evolution, real vision"</a>, 3rd international conference on Simulation of adaptive behavior: from animals to animats 3, pp. 392–401</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Seeing+the+light%3A+Artificial+evolution%2C+real+vision&rft.btitle=3rd+international+conference+on+Simulation+of+adaptive+behavior%3A+from+animals+to+animats+3&rft.pages=392-401&rft.date=1994&rft.aulast=Harvey&rft.aufirst=Inman&rft.au=Husbands%2C+Phil&rft.au=Cliff%2C+Dave&rft_id=https%3A%2F%2Fwww.researchgate.net%2Fpublication%2F229091538_Seeing_the_Light_Artificial_Evolution_Real_Vision&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Evolving_communication_without_dedicated_communication_channels-110"><a href="#cite_ref-Evolving_communication_without_dedicated_communication_channels_110-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFQuinn2001" class="citation conference cs1">Quinn, Matt (2001). "Evolving communication without dedicated communication channels". Advances in Artificial Life: 6th European Conference, ECAL 2001. pp. 357–366. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F3-540-44811-X_38">10.1007/3-540-44811-X_38</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-42567-0" title="Special:BookSources/978-3-540-42567-0"><bdi>978-3-540-42567-0</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Evolving+communication+without+dedicated+communication+channels&rft.btitle=Advances+in+Artificial+Life%3A+6th+European+Conference%2C+ECAL+2001&rft.pages=357-366&rft.date=2001&rft_id=info%3Adoi%2F10.1007%2F3-540-44811-X_38&rft.isbn=978-3-540-42567-0&rft.aulast=Quinn&rft.aufirst=Matt&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-The_dynamics_of_adaptive_behavior:_A_research_program-111"><a href="#cite_ref-The_dynamics_of_adaptive_behavior:_A_research_program_111-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBeer1997" class="citation journal cs1">Beer, Randall D. (1997). "The dynamics of adaptive behavior: A research program". Robotics and Autonomous Systems. 20 (2–4): 257–289. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0921-8890%2896%2900063-2">10.1016/S0921-8890(96)00063-2</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Robotics+and+Autonomous+Systems&rft.atitle=The+dynamics+of+adaptive+behavior%3A+A+research+program&rft.volume=20&rft.issue=2%E2%80%934&rft.pages=257-289&rft.date=1997&rft_id=info%3Adoi%2F10.1016%2FS0921-8890%2896%2900063-2&rft.aulast=Beer&rft.aufirst=Randall+D.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-Sherstinsky-NeurIPS2018-CRACT-3-112"><a href="#cite_ref-Sherstinsky-NeurIPS2018-CRACT-3_112-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSherstinsky2018" class="citation conference cs1">Sherstinsky, Alex (2018-12-07). Bloem-Reddy, Benjamin; Paige, Brooks; Kusner, Matt; Caruana, Rich; Rainforth, Tom; Teh, Yee Whye (eds.). <a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/331718291">Deriving the Recurrent Neural Network Definition and RNN Unrolling Using Signal Processing</a>. <a rel="nofollow" class="external text" href="https://ml-critique-correct.github.io/">Critiquing and Correcting Trends in Machine Learning Workshop at NeurIPS-2018</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.btitle=Deriving+the+Recurrent+Neural+Network+Definition+and+RNN+Unrolling+Using+Signal+Processing&rft.date=2018-12-07&rft.aulast=Sherstinsky&rft.aufirst=Alex&rft_id=https%3A%2F%2Fwww.researchgate.net%2Fpublication%2F331718291&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-113"><a href="#cite_ref-113">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSiegelmannHorneGiles1995" class="citation journal cs1">Siegelmann, Hava T.; Horne, Bill G.; Giles, C. Lee (1995). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=830-HAAACAAJ&pg=PA208">"Computational Capabilities of Recurrent NARX Neural Networks"</a>. IEEE Transactions on Systems, Man, and Cybernetics - Part B: Cybernetics. 27 (2): 208–15. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.48.7468">10.1.1.48.7468</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2F3477.558801">10.1109/3477.558801</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/18255858">18255858</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Systems%2C+Man%2C+and+Cybernetics+-+Part+B%3A+Cybernetics&rft.atitle=Computational+Capabilities+of+Recurrent+NARX+Neural+Networks&rft.volume=27&rft.issue=2&rft.pages=208-15&rft.date=1995&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.48.7468%23id-name%3DCiteSeerX&rft_id=info%3Apmid%2F18255858&rft_id=info%3Adoi%2F10.1109%2F3477.558801&rft.aulast=Siegelmann&rft.aufirst=Hava+T.&rft.au=Horne%2C+Bill+G.&rft.au=Giles%2C+C.+Lee&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3D830-HAAACAAJ%26pg%3DPA208&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-114"><a href="#cite_ref-114">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMiljanovic2012" class="citation journal cs1">Miljanovic, Milos (Feb–Mar 2012). <a rel="nofollow" class="external text" href="http://www.ijcse.com/docs/INDJCSE12-03-01-028.pdf">"Comparative analysis of Recurrent and Finite Impulse Response Neural Networks in Time Series Prediction"</a> (PDF). Indian Journal of Computer and Engineering. 3 (1).</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Indian+Journal+of+Computer+and+Engineering&rft.atitle=Comparative+analysis+of+Recurrent+and+Finite+Impulse+Response+Neural+Networks+in+Time+Series+Prediction&rft.volume=3&rft.issue=1&rft.date=2012-02%2F2012-03&rft.aulast=Miljanovic&rft.aufirst=Milos&rft_id=http%3A%2F%2Fwww.ijcse.com%2Fdocs%2FINDJCSE12-03-01-028.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-115"><a href="#cite_ref-115">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHodassmanMeirKisosBen-Noam2022" class="citation journal cs1">Hodassman, Shiri; Meir, Yuval; Kisos, Karin; Ben-Noam, Itamar; Tugendhaft, Yael; Goldental, Amir; Vardi, Roni; Kanter, Ido (2022-09-29). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC9523036">"Brain inspired neuronal silencing mechanism to enable reliable sequence identification"</a>. Scientific Reports. 12 (1): 16003. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2203.13028">2203.13028</a>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2022NatSR..1216003H">2022NatSR..1216003H</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fs41598-022-20337-x">10.1038/s41598-022-20337-x</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2045-2322">2045-2322</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC9523036">9523036</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/36175466">36175466</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Scientific+Reports&rft.atitle=Brain+inspired+neuronal+silencing+mechanism+to+enable+reliable+sequence+identification&rft.volume=12&rft.issue=1&rft.pages=16003&rft.date=2022-09-29&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC9523036%23id-name%3DPMC&rft_id=info%3Abibcode%2F2022NatSR..1216003H&rft_id=info%3Aarxiv%2F2203.13028&rft.issn=2045-2322&rft_id=info%3Adoi%2F10.1038%2Fs41598-022-20337-x&rft_id=info%3Apmid%2F36175466&rft.aulast=Hodassman&rft.aufirst=Shiri&rft.au=Meir%2C+Yuval&rft.au=Kisos%2C+Karin&rft.au=Ben-Noam%2C+Itamar&rft.au=Tugendhaft%2C+Yael&rft.au=Goldental%2C+Amir&rft.au=Vardi%2C+Roni&rft.au=Kanter%2C+Ido&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC9523036&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-116"><a href="#cite_ref-116">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMetz2016" class="citation news cs1">Metz, Cade (May 18, 2016). <a rel="nofollow" class="external text" href="https://www.wired.com/2016/05/google-tpu-custom-chips/">"Google Built Its Very Own Chips to Power Its AI Bots"</a>. Wired.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Wired&rft.atitle=Google+Built+Its+Very+Own+Chips+to+Power+Its+AI+Bots&rft.date=2016-05-18&rft.aulast=Metz&rft.aufirst=Cade&rft_id=https%3A%2F%2Fwww.wired.com%2F2016%2F05%2Fgoogle-tpu-custom-chips%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-117"><a href="#cite_ref-117">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMayerGomezWierstraNagy2006" class="citation book cs1">Mayer, Hermann; Gomez, Faustino J.; Wierstra, Daan; Nagy, Istvan; Knoll, Alois; Schmidhuber, Jürgen (October 2006). "A System for Robotic Heart Surgery that Learns to Tie Knots Using Recurrent Neural Networks". 2006 IEEE/RSJ International Conference on Intelligent Robots and Systems. pp. 543–548. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.218.3399">10.1.1.218.3399</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FIROS.2006.282190">10.1109/IROS.2006.282190</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-4244-0258-8" title="Special:BookSources/978-1-4244-0258-8"><bdi>978-1-4244-0258-8</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:12284900">12284900</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=A+System+for+Robotic+Heart+Surgery+that+Learns+to+Tie+Knots+Using+Recurrent+Neural+Networks&rft.btitle=2006+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems&rft.pages=543-548&rft.date=2006-10&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.218.3399%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A12284900%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FIROS.2006.282190&rft.isbn=978-1-4244-0258-8&rft.aulast=Mayer&rft.aufirst=Hermann&rft.au=Gomez%2C+Faustino+J.&rft.au=Wierstra%2C+Daan&rft.au=Nagy%2C+Istvan&rft.au=Knoll%2C+Alois&rft.au=Schmidhuber%2C+J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-118"><a href="#cite_ref-118">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWierstraSchmidhuberGomez2005" class="citation conference cs1">Wierstra, Daan; Schmidhuber, Jürgen; Gomez, Faustino J. (2005). <a rel="nofollow" class="external text" href="https://www.academia.edu/5830256">"Evolino: Hybrid Neuroevolution/Optimal Linear Search for Sequence Learning"</a>. Proceedings of the 19th International Joint Conference on Artificial Intelligence (IJCAI), Edinburgh. pp. 853–8. <a href="/wiki/OCLC_(identifier)" class="mw-redirect" title="OCLC (identifier)">OCLC</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/oclc/62330637">62330637</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Evolino%3A+Hybrid+Neuroevolution%2FOptimal+Linear+Search+for+Sequence+Learning&rft.btitle=Proceedings+of+the+19th+International+Joint+Conference+on+Artificial+Intelligence+%28IJCAI%29%2C+Edinburgh&rft.pages=853-8&rft.date=2005&rft_id=info%3Aoclcnum%2F62330637&rft.aulast=Wierstra&rft.aufirst=Daan&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft.au=Gomez%2C+Faustino+J.&rft_id=https%3A%2F%2Fwww.academia.edu%2F5830256&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-119"><a href="#cite_ref-119">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPetneházi2019" class="citation arxiv cs1">Petneházi, Gábor (2019-01-01). "Recurrent neural networks for time series forecasting". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1901.00069">1901.00069</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Recurrent+neural+networks+for+time+series+forecasting&rft.date=2019-01-01&rft_id=info%3Aarxiv%2F1901.00069&rft.aulast=Petneh%C3%A1zi&rft.aufirst=G%C3%A1bor&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-120"><a href="#cite_ref-120">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHewamalageBergmeirBandara2020" class="citation journal cs1">Hewamalage, Hansika; Bergmeir, Christoph; Bandara, Kasun (2020). "Recurrent Neural Networks for Time Series Forecasting: Current Status and Future Directions". International Journal of Forecasting. 37: 388–427. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1909.00590">1909.00590</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ijforecast.2020.06.008">10.1016/j.ijforecast.2020.06.008</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:202540863">202540863</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=International+Journal+of+Forecasting&rft.atitle=Recurrent+Neural+Networks+for+Time+Series+Forecasting%3A+Current+Status+and+Future+Directions&rft.volume=37&rft.pages=388-427&rft.date=2020&rft_id=info%3Aarxiv%2F1909.00590&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A202540863%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1016%2Fj.ijforecast.2020.06.008&rft.aulast=Hewamalage&rft.aufirst=Hansika&rft.au=Bergmeir%2C+Christoph&rft.au=Bandara%2C+Kasun&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-121"><a href="#cite_ref-121">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesSchmidhuber2005" class="citation journal cs1">Graves, Alex; Schmidhuber, Jürgen (2005). "Framewise phoneme classification with bidirectional LSTM and other neural network architectures". Neural Networks. 18 (5–6): 602–610. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.331.5800">10.1.1.331.5800</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.neunet.2005.06.042">10.1016/j.neunet.2005.06.042</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/16112549">16112549</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1856462">1856462</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Networks&rft.atitle=Framewise+phoneme+classification+with+bidirectional+LSTM+and+other+neural+network+architectures&rft.volume=18&rft.issue=5%E2%80%936&rft.pages=602-610&rft.date=2005&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.331.5800%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1856462%23id-name%3DS2CID&rft_id=info%3Apmid%2F16112549&rft_id=info%3Adoi%2F10.1016%2Fj.neunet.2005.06.042&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Schmidhuber%2C+J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-graves2013-122"><a href="#cite_ref-graves2013_122-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesMohamedHinton2013" class="citation conference cs1">Graves, Alex; Mohamed, Abdel-rahman; Hinton, Geoffrey E. (2013). "Speech recognition with deep recurrent neural networks". 2013 IEEE International Conference on Acoustics, Speech and Signal Processing. pp. 6645–9. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1303.5778">1303.5778</a>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2013arXiv1303.5778G">2013arXiv1303.5778G</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FICASSP.2013.6638947">10.1109/ICASSP.2013.6638947</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-4799-0356-6" title="Special:BookSources/978-1-4799-0356-6"><bdi>978-1-4799-0356-6</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:206741496">206741496</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Speech+recognition+with+deep+recurrent+neural+networks&rft.btitle=2013+IEEE+International+Conference+on+Acoustics%2C+Speech+and+Signal+Processing&rft.pages=6645-9&rft.date=2013&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A206741496%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2013arXiv1303.5778G&rft_id=info%3Aarxiv%2F1303.5778&rft_id=info%3Adoi%2F10.1109%2FICASSP.2013.6638947&rft.isbn=978-1-4799-0356-6&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Mohamed%2C+Abdel-rahman&rft.au=Hinton%2C+Geoffrey+E.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-123"><a href="#cite_ref-123">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChangChartierAnumanchipalli2019" class="citation journal cs1">Chang, Edward F.; Chartier, Josh; Anumanchipalli, Gopala K. (24 April 2019). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC9714519">"Speech synthesis from neural decoding of spoken sentences"</a>. Nature. 568 (7753): 493–8. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2019Natur.568..493A">2019Natur.568..493A</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fs41586-019-1119-1">10.1038/s41586-019-1119-1</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1476-4687">1476-4687</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC9714519">9714519</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/31019317">31019317</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:129946122">129946122</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Nature&rft.atitle=Speech+synthesis+from+neural+decoding+of+spoken+sentences&rft.volume=568&rft.issue=7753&rft.pages=493-8&rft.date=2019-04-24&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC9714519%23id-name%3DPMC&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A129946122%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2019Natur.568..493A&rft.issn=1476-4687&rft_id=info%3Adoi%2F10.1038%2Fs41586-019-1119-1&rft_id=info%3Apmid%2F31019317&rft.aulast=Chang&rft.aufirst=Edward+F.&rft.au=Chartier%2C+Josh&rft.au=Anumanchipalli%2C+Gopala+K.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC9714519&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-124"><a href="#cite_ref-124">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMosesMetzgerLiuAnumanchipalli2021" class="citation journal cs1">Moses, David A.; Metzger, Sean L.; Liu, Jessie R.; Anumanchipalli, Gopala K.; Makin, Joseph G.; Sun, Pengfei F.; Chartier, Josh; Dougherty, Maximilian E.; Liu, Patricia M.; Abrams, Gary M.; Tu-Chan, Adelyn; Ganguly, Karunesh; Chang, Edward F. (2021-07-15). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8972947">"Neuroprosthesis for Decoding Speech in a Paralyzed Person with Anarthria"</a>. New England Journal of Medicine. 385 (3): 217–227. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1056%2FNEJMoa2027540">10.1056/NEJMoa2027540</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8972947">8972947</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/34260835">34260835</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=New+England+Journal+of+Medicine&rft.atitle=Neuroprosthesis+for+Decoding+Speech+in+a+Paralyzed+Person+with+Anarthria&rft.volume=385&rft.issue=3&rft.pages=217-227&rft.date=2021-07-15&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC8972947%23id-name%3DPMC&rft_id=info%3Apmid%2F34260835&rft_id=info%3Adoi%2F10.1056%2FNEJMoa2027540&rft.aulast=Moses&rft.aufirst=David+A.&rft.au=Metzger%2C+Sean+L.&rft.au=Liu%2C+Jessie+R.&rft.au=Anumanchipalli%2C+Gopala+K.&rft.au=Makin%2C+Joseph+G.&rft.au=Sun%2C+Pengfei+F.&rft.au=Chartier%2C+Josh&rft.au=Dougherty%2C+Maximilian+E.&rft.au=Liu%2C+Patricia+M.&rft.au=Abrams%2C+Gary+M.&rft.au=Tu-Chan%2C+Adelyn&rft.au=Ganguly%2C+Karunesh&rft.au=Chang%2C+Edward+F.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC8972947&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-125"><a href="#cite_ref-125">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMalhotraVigShroffAgarwal2015" class="citation conference cs1">Malhotra, Pankaj; Vig, Lovekesh; Shroff, Gautam; Agarwal, Puneet (April 2015). <a rel="nofollow" class="external text" href="https://books.google.com/books?id=USGLCgAAQBAJ&pg=PA89">"Long Short Term Memory Networks for Anomaly Detection in Time Series"</a>. European Symposium on Artificial Neural Networks, Computational Intelligence and Machine Learning – ESANN 2015. Ciaco. pp. 89–94. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-2-87587-015-5" title="Special:BookSources/978-2-87587-015-5"><bdi>978-2-87587-015-5</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Long+Short+Term+Memory+Networks+for+Anomaly+Detection+in+Time+Series&rft.btitle=European+Symposium+on+Artificial+Neural+Networks%2C+Computational+Intelligence+and+Machine+Learning+%E2%80%93+ESANN+2015&rft.pages=89-94&rft.pub=Ciaco&rft.date=2015-04&rft.isbn=978-2-87587-015-5&rft.aulast=Malhotra&rft.aufirst=Pankaj&rft.au=Vig%2C+Lovekesh&rft.au=Shroff%2C+Gautam&rft.au=Agarwal%2C+Puneet&rft_id=https%3A%2F%2Fbooks.google.com%2Fbooks%3Fid%3DUSGLCgAAQBAJ%26pg%3DPA89&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-126"><a href="#cite_ref-126">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://paperswithcode.com/paper/deephs-hdrvideo-deep-high-speed-high-dynamic">"Papers with Code - DeepHS-HDRVideo: Deep High Speed High Dynamic Range Video Reconstruction"</a>. paperswithcode.com. Retrieved 2022-10-13.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=paperswithcode.com&rft.atitle=Papers+with+Code+-+DeepHS-HDRVideo%3A+Deep+High+Speed+High+Dynamic+Range+Video+Reconstruction&rft_id=https%3A%2F%2Fpaperswithcode.com%2Fpaper%2Fdeephs-hdrvideo-deep-high-speed-high-dynamic&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-peephole2002-127"><a href="#cite_ref-peephole2002_127-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGersSchraudolphSchmidhuber2002" class="citation journal cs1">Gers, Felix A.; Schraudolph, Nicol N.; Schmidhuber, Jürgen (2002). <a rel="nofollow" class="external text" href="http://www.jmlr.org/papers/volume3/gers02a/gers02a.pdf">"Learning precise timing with LSTM recurrent networks"</a> (PDF). Journal of Machine Learning Research. 3: 115–143.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Machine+Learning+Research&rft.atitle=Learning+precise+timing+with+LSTM+recurrent+networks&rft.volume=3&rft.pages=115-143&rft.date=2002&rft.aulast=Gers&rft.aufirst=Felix+A.&rft.au=Schraudolph%2C+Nicol+N.&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fwww.jmlr.org%2Fpapers%2Fvolume3%2Fgers02a%2Fgers02a.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-128"><a href="#cite_ref-128">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFEckSchmidhuber2002" class="citation book cs1">Eck, Douglas; Schmidhuber, Jürgen (2002-08-28). "Learning the Long-Term Structure of the Blues". Artificial Neural Networks — ICANN 2002. Lecture Notes in Computer Science. Vol. 2415. Berlin, Heidelberg: Springer. pp. 284–289. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.116.3620">10.1.1.116.3620</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F3-540-46084-5_47">10.1007/3-540-46084-5_47</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-46084-8" title="Special:BookSources/978-3-540-46084-8"><bdi>978-3-540-46084-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Learning+the+Long-Term+Structure+of+the+Blues&rft.btitle=Artificial+Neural+Networks+%E2%80%94+ICANN+2002&rft.place=Berlin%2C+Heidelberg&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=284-289&rft.pub=Springer&rft.date=2002-08-28&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.116.3620%23id-name%3DCiteSeerX&rft_id=info%3Adoi%2F10.1007%2F3-540-46084-5_47&rft.isbn=978-3-540-46084-8&rft.aulast=Eck&rft.aufirst=Douglas&rft.au=Schmidhuber%2C+J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-129"><a href="#cite_ref-129">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSchmidhuberGersEck2002" class="citation journal cs1">Schmidhuber, Jürgen; Gers, Felix A.; Eck, Douglas (2002). "Learning nonregular languages: A comparison of simple recurrent networks and LSTM". Neural Computation. 14 (9): 2039–2041. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.11.7369">10.1.1.11.7369</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2F089976602320263980">10.1162/089976602320263980</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/12184841">12184841</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:30459046">30459046</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Computation&rft.atitle=Learning+nonregular+languages%3A+A+comparison+of+simple+recurrent+networks+and+LSTM&rft.volume=14&rft.issue=9&rft.pages=2039-2041&rft.date=2002&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.11.7369%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A30459046%23id-name%3DS2CID&rft_id=info%3Apmid%2F12184841&rft_id=info%3Adoi%2F10.1162%2F089976602320263980&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rft.au=Gers%2C+Felix+A.&rft.au=Eck%2C+Douglas&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-130"><a href="#cite_ref-130">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPérez-OrtizGersEckSchmidhuber2003" class="citation journal cs1">Pérez-Ortiz, Juan Antonio; Gers, Felix A.; Eck, Douglas; Schmidhuber, Jürgen (2003). "Kalman filters improve LSTM network performance in problems unsolvable by traditional recurrent nets". Neural Networks. 16 (2): 241–250. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.381.1992">10.1.1.381.1992</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fs0893-6080%2802%2900219-8">10.1016/s0893-6080(02)00219-8</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/12628609">12628609</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Networks&rft.atitle=Kalman+filters+improve+LSTM+network+performance+in+problems+unsolvable+by+traditional+recurrent+nets&rft.volume=16&rft.issue=2&rft.pages=241-250&rft.date=2003&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.381.1992%23id-name%3DCiteSeerX&rft_id=info%3Apmid%2F12628609&rft_id=info%3Adoi%2F10.1016%2Fs0893-6080%2802%2900219-8&rft.aulast=P%C3%A9rez-Ortiz&rft.aufirst=Juan+Antonio&rft.au=Gers%2C+Felix+A.&rft.au=Eck%2C+Douglas&rft.au=Schmidhuber%2C+J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-131"><a href="#cite_ref-131">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesSchmidhuber2009" class="citation conference cs1 cs1-prop-long-vol">Graves, Alex; Schmidhuber, Jürgen (2009). <a rel="nofollow" class="external text" href="http://papers.neurips.cc/paper/3449-offline-handwriting-recognition-with-multidimensional-recurrent-neural-networks.pdf">"Offline Handwriting Recognition with Multidimensional Recurrent Neural Networks"</a> (PDF). Advances in Neural Information Processing Systems. Vol. 22, NIPS'22. MIT Press. pp. 545–552.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Offline+Handwriting+Recognition+with+Multidimensional+Recurrent+Neural+Networks&rft.btitle=Advances+in+Neural+Information+Processing+Systems&rft.pages=545-552&rft.pub=MIT+Press&rft.date=2009&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fpapers.neurips.cc%2Fpaper%2F3449-offline-handwriting-recognition-with-multidimensional-recurrent-neural-networks.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-132"><a href="#cite_ref-132">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGravesFernándezLiwickiBunke2007" class="citation conference cs1">Graves, Alex; Fernández, Santiago; Liwicki, Marcus; Bunke, Horst; Schmidhuber, Jürgen (2007). <a rel="nofollow" class="external text" href="http://dl.acm.org/citation.cfm?id=2981562.2981635">"Unconstrained Online Handwriting Recognition with Recurrent Neural Networks"</a>. Proceedings of the 20th International Conference on Neural Information Processing Systems. Curran Associates. pp. 577–584. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-60560-352-0" title="Special:BookSources/978-1-60560-352-0"><bdi>978-1-60560-352-0</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Unconstrained+Online+Handwriting+Recognition+with+Recurrent+Neural+Networks&rft.btitle=Proceedings+of+the+20th+International+Conference+on+Neural+Information+Processing+Systems&rft.pages=577-584&rft.pub=Curran+Associates&rft.date=2007&rft.isbn=978-1-60560-352-0&rft.aulast=Graves&rft.aufirst=Alex&rft.au=Fern%C3%A1ndez%2C+Santiago&rft.au=Liwicki%2C+Marcus&rft.au=Bunke%2C+Horst&rft.au=Schmidhuber%2C+J%C3%BCrgen&rft_id=http%3A%2F%2Fdl.acm.org%2Fcitation.cfm%3Fid%3D2981562.2981635&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-133"><a href="#cite_ref-133">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBaccoucheMamaletWolfGarcia2011" class="citation book cs1">Baccouche, Moez; Mamalet, Franck; Wolf, Christian; Garcia, Christophe; Baskurt, Atilla (2011). "Sequential Deep Learning for Human Action Recognition". In Salah, Albert Ali; Lepri, Bruno (eds.). Human Behavior Unterstanding. Lecture Notes in Computer Science. Vol. 7065. Amsterdam, Netherlands: Springer. pp. 29–39. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-642-25446-8_4">10.1007/978-3-642-25446-8_4</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-642-25445-1" title="Special:BookSources/978-3-642-25445-1"><bdi>978-3-642-25445-1</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Sequential+Deep+Learning+for+Human+Action+Recognition&rft.btitle=Human+Behavior+Unterstanding&rft.place=Amsterdam%2C+Netherlands&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=29-39&rft.pub=Springer&rft.date=2011&rft_id=info%3Adoi%2F10.1007%2F978-3-642-25446-8_4&rft.isbn=978-3-642-25445-1&rft.aulast=Baccouche&rft.aufirst=Moez&rft.au=Mamalet%2C+Franck&rft.au=Wolf%2C+Christian&rft.au=Garcia%2C+Christophe&rft.au=Baskurt%2C+Atilla&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-134"><a href="#cite_ref-134">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHochreiterHeuselObermayer2007" class="citation journal cs1">Hochreiter, Sepp; Heusel, Martin; Obermayer, Klaus (2007). <a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fbioinformatics%2Fbtm247">"Fast model-based protein homology detection without alignment"</a>. Bioinformatics. 23 (14): 1728–1736. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fbioinformatics%2Fbtm247">10.1093/bioinformatics/btm247</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/17488755">17488755</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Bioinformatics&rft.atitle=Fast+model-based+protein+homology+detection+without+alignment&rft.volume=23&rft.issue=14&rft.pages=1728-1736&rft.date=2007&rft_id=info%3Adoi%2F10.1093%2Fbioinformatics%2Fbtm247&rft_id=info%3Apmid%2F17488755&rft.aulast=Hochreiter&rft.aufirst=Sepp&rft.au=Heusel%2C+Martin&rft.au=Obermayer%2C+Klaus&rft_id=https%3A%2F%2Fdoi.org%2F10.1093%252Fbioinformatics%252Fbtm247&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-ThireoReczko-135"><a href="#cite_ref-ThireoReczko_135-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFThireouReczko2007" class="citation journal cs1">Thireou, Trias; Reczko, Martin (July 2007). "Bidirectional Long Short-Term Memory Networks for Predicting the Subcellular Localization of Eukaryotic Proteins". IEEE/ACM Transactions on Computational Biology and Bioinformatics. 4 (3): 441–446. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2Ftcbb.2007.1015">10.1109/tcbb.2007.1015</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/17666763">17666763</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:11787259">11787259</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE%2FACM+Transactions+on+Computational+Biology+and+Bioinformatics&rft.atitle=Bidirectional+Long+Short-Term+Memory+Networks+for+Predicting+the+Subcellular+Localization+of+Eukaryotic+Proteins&rft.volume=4&rft.issue=3&rft.pages=441-446&rft.date=2007-07&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A11787259%23id-name%3DS2CID&rft_id=info%3Apmid%2F17666763&rft_id=info%3Adoi%2F10.1109%2Ftcbb.2007.1015&rft.aulast=Thireou&rft.aufirst=Trias&rft.au=Reczko%2C+Martin&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-136"><a href="#cite_ref-136">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFTaxVerenichLa_RosaDumas2017" class="citation book cs1">Tax, Niek; Verenich, Ilya; La Rosa, Marcello; Dumas, Marlon (2017). "Predictive Business Process Monitoring with LSTM Neural Networks". Advanced Information Systems Engineering. Lecture Notes in Computer Science. Vol. 10253. pp. 477–492. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1612.02130">1612.02130</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-319-59536-8_30">10.1007/978-3-319-59536-8_30</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-319-59535-1" title="Special:BookSources/978-3-319-59535-1"><bdi>978-3-319-59535-1</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:2192354">2192354</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Predictive+Business+Process+Monitoring+with+LSTM+Neural+Networks&rft.btitle=Advanced+Information+Systems+Engineering&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=477-492&rft.date=2017&rft_id=info%3Aarxiv%2F1612.02130&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A2192354%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1007%2F978-3-319-59536-8_30&rft.isbn=978-3-319-59535-1&rft.aulast=Tax&rft.aufirst=Niek&rft.au=Verenich%2C+Ilya&rft.au=La+Rosa%2C+Marcello&rft.au=Dumas%2C+Marlon&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-137"><a href="#cite_ref-137">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChoiBahadoriSchuetzStewart2016" class="citation journal cs1">Choi, Edward; Bahadori, Mohammad Taha; Schuetz, Andy; Stewart, Walter F.; Sun, Jimeng (2016). <a rel="nofollow" class="external text" href="http://proceedings.mlr.press/v56/Choi16.html">"Doctor AI: Predicting Clinical Events via Recurrent Neural Networks"</a>. JMLR Workshop and Conference Proceedings. 56: 301–318. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1511.05942">1511.05942</a>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2015arXiv151105942C">2015arXiv151105942C</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5341604">5341604</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/28286600">28286600</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=JMLR+Workshop+and+Conference+Proceedings&rft.atitle=Doctor+AI%3A+Predicting+Clinical+Events+via+Recurrent+Neural+Networks&rft.volume=56&rft.pages=301-318&rft.date=2016&rft_id=info%3Aarxiv%2F1511.05942&rft_id=info%3Apmid%2F28286600&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC5341604%23id-name%3DPMC&rft_id=info%3Abibcode%2F2015arXiv151105942C&rft.aulast=Choi&rft.aufirst=Edward&rft.au=Bahadori%2C+Mohammad+Taha&rft.au=Schuetz%2C+Andy&rft.au=Stewart%2C+Walter+F.&rft.au=Sun%2C+Jimeng&rft_id=http%3A%2F%2Fproceedings.mlr.press%2Fv56%2FChoi16.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> <li id="cite_note-138"><a href="#cite_ref-138">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.princeton.edu/news/2017/12/15/artificial-intelligence-helps-accelerate-progress-toward-efficient-fusion-reactions">"Artificial intelligence helps accelerate progress toward efficient fusion reactions"</a>. Princeton University. Retrieved 2023-06-12.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=Princeton+University&rft.atitle=Artificial+intelligence+helps+accelerate+progress+toward+efficient+fusion+reactions&rft_id=https%3A%2F%2Fwww.princeton.edu%2Fnews%2F2017%2F12%2F15%2Fartificial-intelligence-helps-accelerate-progress-toward-efficient-fusion-reactions&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"> </li> </ol></div> <div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2>[<a href="/w/index.php?title=Recurrent_neural_network&action=edit&section=37" title="Edit section: Further reading">edit</a>]</div> <ul><li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMandicChambers2001" class="citation book cs1">Mandic, Danilo P.; Chambers, Jonathon A. (2001). Recurrent Neural Networks for Prediction: Learning Algorithms, Architectures and Stability. Wiley. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-471-49517-8" title="Special:BookSources/978-0-471-49517-8"><bdi>978-0-471-49517-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Recurrent+Neural+Networks+for+Prediction%3A+Learning+Algorithms%2C+Architectures+and+Stability&rft.pub=Wiley&rft.date=2001&rft.isbn=978-0-471-49517-8&rft.aulast=Mandic&rft.aufirst=Danilo+P.&rft.au=Chambers%2C+Jonathon+A.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGrossberg2013" class="citation journal cs1">Grossberg, Stephen (2013-02-22). <a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.1888">"Recurrent Neural Networks"</a>. Scholarpedia. 8 (2): 1888. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2013SchpJ...8.1888G">2013SchpJ...8.1888G</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.1888">10.4249/scholarpedia.1888</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1941-6016">1941-6016</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Scholarpedia&rft.atitle=Recurrent+Neural+Networks&rft.volume=8&rft.issue=2&rft.pages=1888&rft.date=2013-02-22&rft.issn=1941-6016&rft_id=info%3Adoi%2F10.4249%2Fscholarpedia.1888&rft_id=info%3Abibcode%2F2013SchpJ...8.1888G&rft.aulast=Grossberg&rft.aufirst=Stephen&rft_id=https%3A%2F%2Fdoi.org%2F10.4249%252Fscholarpedia.1888&rfr_id=info%3Asid%2Fen.wikipedia.org%3ARecurrent+neural+network" class="Z3988"></li> <li><a rel="nofollow" class="external text" href="http://www.idsia.ch/~juergen/rnn.html">Recurrent Neural Networks</a>. List of RNN papers by <a href="/wiki/J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a>'s group at <a href="/wiki/Dalle_Molle_Institute_for_Artificial_Intelligence_Research" title="Dalle Molle Institute for Artificial Intelligence Research">Dalle Molle Institute for Artificial Intelligence Research</a>.</li></ul> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="Artificial_intelligence" style="padding:3px"><table class="nowraplinks hlist mw-collapsible {{{state}}} navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1239400231"><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Artificial_intelligence_(AI)" title="Template:Artificial intelligence (AI)"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Artificial_intelligence_(AI)" class="mw-redirect" title="Template talk:Artificial intelligence (AI)"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Artificial_intelligence_(AI)" title="Special:EditPage/Template:Artificial intelligence (AI)"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Artificial_intelligence" style="font-size:114%;margin:0 4em"><a href="/wiki/Artificial_intelligence" title="Artificial intelligence">Artificial intelligence</a></div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Parameter" title="Parameter">Parameter</a> <ul><li><a href="/wiki/Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">Hyperparameter</a></li></ul></li> <li><a href="/wiki/Loss_functions_for_classification" title="Loss functions for classification">Loss functions</a></li> <li><a href="/wiki/Regression_analysis" title="Regression analysis">Regression</a> <ul><li><a href="/wiki/Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li> <li><a href="/wiki/Double_descent" title="Double descent">Double descent</a></li> <li><a href="/wiki/Overfitting" title="Overfitting">Overfitting</a></li></ul></li> <li><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></li> <li><a href="/wiki/Gradient_descent" title="Gradient descent">Gradient descent</a> <ul><li><a href="/wiki/Stochastic_gradient_descent" title="Stochastic gradient descent">SGD</a></li> <li><a href="/wiki/Quasi-Newton_method" title="Quasi-Newton method">Quasi-Newton method</a></li> <li><a href="/wiki/Conjugate_gradient_method" title="Conjugate gradient method">Conjugate gradient method</a></li></ul></li> <li><a href="/wiki/Backpropagation" title="Backpropagation">Backpropagation</a></li> <li><a href="/wiki/Attention_(machine_learning)" title="Attention (machine learning)">Attention</a></li> <li><a href="/wiki/Convolution" title="Convolution">Convolution</a></li> <li><a href="/wiki/Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization</a> <ul><li><a href="/wiki/Batch_normalization" title="Batch normalization">Batchnorm</a></li></ul></li> <li><a href="/wiki/Activation_function" title="Activation function">Activation</a> <ul><li><a href="/wiki/Softmax_function" title="Softmax function">Softmax</a></li> <li><a href="/wiki/Sigmoid_function" title="Sigmoid function">Sigmoid</a></li> <li><a href="/wiki/Rectifier_(neural_networks)" title="Rectifier (neural networks)">Rectifier</a></li></ul></li> <li><a href="/wiki/Gating_mechanism" title="Gating mechanism">Gating</a></li> <li><a href="/wiki/Weight_initialization" title="Weight initialization">Weight initialization</a></li> <li><a href="/wiki/Regularization_(mathematics)" title="Regularization (mathematics)">Regularization</a></li> <li><a href="/wiki/Training,_validation,_and_test_data_sets" title="Training, validation, and test data sets">Datasets</a> <ul><li><a href="/wiki/Data_augmentation" title="Data augmentation">Augmentation</a></li></ul></li> <li><a href="/wiki/Prompt_engineering" title="Prompt engineering">Prompt engineering</a></li> <li><a href="/wiki/Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a> <ul><li><a href="/wiki/Q-learning" title="Q-learning">Q-learning</a></li> <li><a href="/wiki/State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li> <li><a href="/wiki/Imitation_learning" title="Imitation learning">Imitation</a></li></ul></li> <li><a href="/wiki/Diffusion_process" title="Diffusion process">Diffusion</a></li> <li><a href="/wiki/Latent_diffusion_model" title="Latent diffusion model">Latent diffusion model</a></li> <li><a href="/wiki/Autoregressive_model" title="Autoregressive model">Autoregression</a></li> <li><a href="/wiki/Adversarial_machine_learning" title="Adversarial machine learning">Adversary</a></li> <li><a href="/wiki/Retrieval-augmented_generation" title="Retrieval-augmented generation">RAG</a></li> <li><a href="/wiki/Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">RLHF</a></li> <li><a href="/wiki/Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li> <li><a href="/wiki/Word_embedding" title="Word embedding">Word embedding</a></li> <li><a href="/wiki/Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Applications</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Machine_learning" title="Machine learning">Machine learning</a> <ul><li><a href="/wiki/Prompt_engineering#In-context_learning" title="Prompt engineering">In-context learning</a></li></ul></li> <li><a href="/wiki/Neural_network_(machine_learning)" title="Neural network (machine learning)">Artificial neural network</a> <ul><li><a href="/wiki/Deep_learning" title="Deep learning">Deep learning</a></li></ul></li> <li><a href="/wiki/Language_model" title="Language model">Language model</a> <ul><li><a href="/wiki/Large_language_model" title="Large language model">Large language model</a></li> <li><a href="/wiki/Neural_machine_translation" title="Neural machine translation">NMT</a></li></ul></li> <li><a href="/wiki/Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Implementations</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Audio–visual</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/AlexNet" title="AlexNet">AlexNet</a></li> <li><a href="/wiki/WaveNet" title="WaveNet">WaveNet</a></li> <li><a href="/wiki/Human_image_synthesis" title="Human image synthesis">Human image synthesis</a></li> <li><a href="/wiki/Handwriting_recognition" title="Handwriting recognition">HWR</a></li> <li><a href="/wiki/Optical_character_recognition" title="Optical character recognition">OCR</a></li> <li><a href="/wiki/Deep_learning_speech_synthesis" title="Deep learning speech synthesis">Speech synthesis</a> <ul><li><a href="/wiki/ElevenLabs" title="ElevenLabs">ElevenLabs</a></li></ul></li> <li><a href="/wiki/Speech_recognition" title="Speech recognition">Speech recognition</a> <ul><li><a href="/wiki/Whisper_(speech_recognition_system)" title="Whisper (speech recognition system)">Whisper</a></li></ul></li> <li><a href="/wiki/Facial_recognition_system" title="Facial recognition system">Facial recognition</a></li> <li><a href="/wiki/AlphaFold" title="AlphaFold">AlphaFold</a></li> <li><a href="/wiki/Text-to-image_model" title="Text-to-image model">Text-to-image models</a> <ul><li><a href="/wiki/DALL-E" title="DALL-E">DALL-E</a></li> <li><a href="/wiki/Flux_(text-to-image_model)" title="Flux (text-to-image model)">Flux</a></li> <li><a href="/wiki/Ideogram_(text-to-image_model)" title="Ideogram (text-to-image model)">Ideogram</a></li> <li><a href="/wiki/Midjourney" title="Midjourney">Midjourney</a></li> <li><a href="/wiki/Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a></li></ul></li> <li><a href="/wiki/Text-to-video_model" title="Text-to-video model">Text-to-video models</a> <ul><li><a href="/wiki/Sora_(text-to-video_model)" title="Sora (text-to-video model)">Sora</a></li> <li><a href="/wiki/Dream_Machine_(text-to-video_model)" title="Dream Machine (text-to-video model)">Dream Machine</a></li> <li><a href="/wiki/VideoPoet" title="VideoPoet">VideoPoet</a></li></ul></li> <li><a href="/wiki/Music_and_artificial_intelligence" title="Music and artificial intelligence">Music generation</a> <ul><li><a href="/wiki/Suno_AI" title="Suno AI">Suno AI</a></li> <li><a href="/wiki/Udio" title="Udio">Udio</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Text</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Word2vec" title="Word2vec">Word2vec</a></li> <li><a href="/wiki/Seq2seq" title="Seq2seq">Seq2seq</a></li> <li><a href="/wiki/GloVe" title="GloVe">GloVe</a></li> <li><a href="/wiki/BERT_(language_model)" title="BERT (language model)">BERT</a></li> <li><a href="/wiki/T5_(language_model)" title="T5 (language model)">T5</a></li> <li><a href="/wiki/Llama_(language_model)" title="Llama (language model)">Llama</a></li> <li><a href="/wiki/Chinchilla_(language_model)" title="Chinchilla (language model)">Chinchilla AI</a></li> <li><a href="/wiki/PaLM" title="PaLM">PaLM</a></li> <li><a href="/wiki/Generative_pre-trained_transformer" title="Generative pre-trained transformer">GPT</a> <ul><li><a href="/wiki/GPT-1" title="GPT-1">1</a></li> <li><a href="/wiki/GPT-2" title="GPT-2">2</a></li> <li><a href="/wiki/GPT-3" title="GPT-3">3</a></li> <li><a href="/wiki/GPT-J" title="GPT-J">J</a></li> <li><a href="/wiki/ChatGPT" title="ChatGPT">ChatGPT</a></li> <li><a href="/wiki/GPT-4" title="GPT-4">4</a></li> <li><a href="/wiki/GPT-4o" title="GPT-4o">4o</a></li> <li><a href="/wiki/OpenAI_o1" title="OpenAI o1">o1</a></li></ul></li> <li><a href="/wiki/Claude_(language_model)" title="Claude (language model)">Claude</a></li> <li><a href="/wiki/Gemini_(language_model)" title="Gemini (language model)">Gemini</a></li> <li><a href="/wiki/Grok_(chatbot)" title="Grok (chatbot)">Grok</a></li> <li><a href="/wiki/LaMDA" title="LaMDA">LaMDA</a></li> <li><a href="/wiki/BLOOM_(language_model)" title="BLOOM (language model)">BLOOM</a></li> <li><a href="/wiki/Project_Debater" title="Project Debater">Project Debater</a></li> <li><a href="/wiki/IBM_Watson" title="IBM Watson">IBM Watson</a></li> <li><a href="/wiki/IBM_Watsonx" title="IBM Watsonx">IBM Watsonx</a></li> <li><a href="/wiki/IBM_Granite" title="IBM Granite">Granite</a></li> <li><a href="/wiki/Huawei_PanGu" title="Huawei PanGu">PanGu-Σ</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Decisional</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/AlphaGo" title="AlphaGo">AlphaGo</a></li> <li><a href="/wiki/AlphaZero" title="AlphaZero">AlphaZero</a></li> <li><a href="/wiki/OpenAI_Five" title="OpenAI Five">OpenAI Five</a></li> <li><a href="/wiki/Self-driving_car" title="Self-driving car">Self-driving car</a></li> <li><a href="/wiki/MuZero" title="MuZero">MuZero</a></li> <li><a href="/wiki/Action_selection" title="Action selection">Action selection</a> <ul><li><a href="/wiki/AutoGPT" title="AutoGPT">AutoGPT</a></li></ul></li> <li><a href="/wiki/Robot_control" title="Robot control">Robot control</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Alan_Turing" title="Alan Turing">Alan Turing</a></li> <li><a href="/wiki/Warren_Sturgis_McCulloch" title="Warren Sturgis McCulloch">Warren Sturgis McCulloch</a></li> <li><a href="/wiki/Walter_Pitts" title="Walter Pitts">Walter Pitts</a></li> <li><a href="/wiki/John_von_Neumann" title="John von Neumann">John von Neumann</a></li> <li><a href="/wiki/Claude_Shannon" title="Claude Shannon">Claude Shannon</a></li> <li><a href="/wiki/Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a></li> <li><a href="/wiki/John_McCarthy_(computer_scientist)" title="John McCarthy (computer scientist)">John McCarthy</a></li> <li><a href="/wiki/Nathaniel_Rochester_(computer_scientist)" title="Nathaniel Rochester (computer scientist)">Nathaniel Rochester</a></li> <li><a href="/wiki/Allen_Newell" title="Allen Newell">Allen Newell</a></li> <li><a href="/wiki/Cliff_Shaw" title="Cliff Shaw">Cliff Shaw</a></li> <li><a href="/wiki/Herbert_A._Simon" title="Herbert A. Simon">Herbert A. Simon</a></li> <li><a href="/wiki/Oliver_Selfridge" title="Oliver Selfridge">Oliver Selfridge</a></li> <li><a href="/wiki/Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a></li> <li><a href="/wiki/Bernard_Widrow" title="Bernard Widrow">Bernard Widrow</a></li> <li><a href="/wiki/Joseph_Weizenbaum" title="Joseph Weizenbaum">Joseph Weizenbaum</a></li> <li><a href="/wiki/Seymour_Papert" title="Seymour Papert">Seymour Papert</a></li> <li><a href="/wiki/Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a></li> <li><a href="/wiki/Paul_Werbos" title="Paul Werbos">Paul Werbos</a></li> <li><a href="/wiki/J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a></li> <li><a href="/wiki/Yann_LeCun" title="Yann LeCun">Yann LeCun</a></li> <li><a href="/wiki/Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li> <li><a href="/wiki/John_Hopfield" title="John Hopfield">John Hopfield</a></li> <li><a href="/wiki/Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li> <li><a href="/wiki/Lotfi_A._Zadeh" title="Lotfi A. Zadeh">Lotfi A. Zadeh</a></li> <li><a href="/wiki/Stephen_Grossberg" title="Stephen Grossberg">Stephen Grossberg</a></li> <li><a href="/wiki/Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Alex Graves</a></li> <li><a href="/wiki/Andrew_Ng" title="Andrew Ng">Andrew Ng</a></li> <li><a href="/wiki/Fei-Fei_Li" title="Fei-Fei Li">Fei-Fei Li</a></li> <li><a href="/wiki/Alex_Krizhevsky" title="Alex Krizhevsky">Alex Krizhevsky</a></li> <li><a href="/wiki/Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li> <li><a href="/wiki/Demis_Hassabis" title="Demis Hassabis">Demis Hassabis</a></li> <li><a href="/wiki/David_Silver_(computer_scientist)" title="David Silver (computer scientist)">David Silver</a></li> <li><a href="/wiki/Ian_Goodfellow" title="Ian Goodfellow">Ian Goodfellow</a></li> <li><a href="/wiki/Andrej_Karpathy" title="Andrej Karpathy">Andrej Karpathy</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Architectures</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a></li> <li><a href="/wiki/Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></li> <li><a href="/wiki/Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a> <ul><li><a href="/wiki/Vision_transformer" title="Vision transformer">Vision transformer (ViT)</a></li></ul></li> <li><a class="mw-selflink selflink">Recurrent neural network (RNN)</a></li> <li><a href="/wiki/Long_short-term_memory" title="Long short-term memory">Long short-term memory (LSTM)</a></li> <li><a href="/wiki/Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit (GRU)</a></li> <li><a href="/wiki/Echo_state_network" title="Echo state network">Echo state network</a></li> <li><a href="/wiki/Multilayer_perceptron" title="Multilayer perceptron">Multilayer perceptron (MLP)</a></li> <li><a href="/wiki/Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network (CNN)</a></li> <li><a href="/wiki/Residual_neural_network" title="Residual neural network">Residual neural network (RNN)</a></li> <li><a href="/wiki/Highway_network" title="Highway network">Highway network</a></li> <li><a href="/wiki/Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li> <li><a href="/wiki/Autoencoder" title="Autoencoder">Autoencoder</a></li> <li><a href="/wiki/Variational_autoencoder" title="Variational autoencoder">Variational autoencoder (VAE)</a></li> <li><a href="/wiki/Generative_adversarial_network" title="Generative adversarial network">Generative adversarial network (GAN)</a></li> <li><a href="/wiki/Graph_neural_network" title="Graph neural network">Graph neural network (GNN)</a></li></ul> </div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><a href="/wiki/File:Symbol_portal_class.svg" class="mw-file-description" title="Portal"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/16px-Symbol_portal_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/23px-Symbol_portal_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/31px-Symbol_portal_class.svg.png 2x" data-file-width="180" data-file-height="185" /></a> Portals <ul><li><a href="/wiki/Portal:Technology" title="Portal:Technology">Technology</a></li></ul></li> <li><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/16px-Symbol_category_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/23px-Symbol_category_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/31px-Symbol_category_class.svg.png 2x" data-file-width="180" data-file-height="185" /> Categories <ul><li><a href="/wiki/Category:Artificial_neural_networks" title="Category:Artificial neural networks">Artificial neural networks</a></li> <li><a href="/wiki/Category:Machine_learning" title="Category:Machine learning">Machine learning</a></li></ul></li> <li><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/d/db/Symbol_list_class.svg/16px-Symbol_list_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/d/db/Symbol_list_class.svg/23px-Symbol_list_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/d/db/Symbol_list_class.svg/31px-Symbol_list_class.svg.png 2x" data-file-width="180" data-file-height="185" /> List <ul><li><a href="/wiki/List_of_artificial_intelligence_companies" title="List of artificial intelligence companies">Companies</a></li> <li><a href="/wiki/List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235"></div><div role="navigation" class="navbox authority-control" aria-label="Navbox" style="padding:3px"><table class="nowraplinks hlist navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Help:Authority_control" title="Help:Authority control">Authority control databases</a>: National <a href="https://www.wikidata.org/wiki/Q1457734#identifiers" title="Edit this at Wikidata"><img alt="Edit this at Wikidata" src="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/10px-OOjs_UI_icon_edit-ltr-progressive.svg.png" decoding="async" width="10" height="10" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/15px-OOjs_UI_icon_edit-ltr-progressive.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/20px-OOjs_UI_icon_edit-ltr-progressive.svg.png 2x" data-file-width="20" data-file-height="20" /></a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"><ul><li><a rel="nofollow" class="external text" href="https://d-nb.info/gnd/4379549-3">Germany</a></li></ul></div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1&useformat=desktop" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Recurrent_neural_network&oldid=1257922012">https://en.wikipedia.org/w/index.php?title=Recurrent_neural_network&oldid=1257922012</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Category</a>: <ul><li><a href="/wiki/Category:Neural_network_architectures" title="Category:Neural network architectures">Neural network architectures</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:CS1:_long_volume_value" title="Category:CS1: long volume value">CS1: long volume value</a></li><li><a href="/wiki/Category:All_articles_with_dead_external_links" title="Category:All articles with dead external links">All articles with dead external links</a></li><li><a href="/wiki/Category:Articles_with_dead_external_links_from_June_2024" title="Category:Articles with dead external links from June 2024">Articles with dead external links from June 2024</a></li><li><a href="/wiki/Category:Articles_with_permanently_dead_external_links" title="Category:Articles with permanently dead external links">Articles with permanently dead external links</a></li><li><a href="/wiki/Category:CS1_Finnish-language_sources_(fi)" title="Category:CS1 Finnish-language sources (fi)">CS1 Finnish-language sources (fi)</a></li><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_is_different_from_Wikidata" title="Category:Short description is different from Wikidata">Short description is different from Wikidata</a></li><li><a href="/wiki/Category:All_articles_with_unsourced_statements" title="Category:All articles with unsourced statements">All articles with unsourced statements</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_June_2017" title="Category:Articles with unsourced statements from June 2017">Articles with unsourced statements from June 2017</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 17 November 2024, at 07:27 (UTC).</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Recurrent_neural_network&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-5c59558b9d-v2f7m","wgBackendResponseTime":141,"wgPageParseReport":{"limitreport":{"cputime":"1.830","walltime":"2.120","ppvisitednodes":{"value":10705,"limit":1000000},"postexpandincludesize":{"value":406347,"limit":2097152},"templateargumentsize":{"value":6479,"limit":2097152},"expansiondepth":{"value":17,"limit":100},"expensivefunctioncount":{"value":18,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":559414,"limit":5000000},"entityaccesscount":{"value":2,"limit":400},"timingprofile":["100.00% 1717.661 1 -total"," 68.90% 1183.471 1 Template:Reflist"," 32.07% 550.839 69 Template:Cite_journal"," 7.99% 137.207 21 Template:Cite_book"," 6.52% 111.972 1 Template:Machine_learning"," 6.19% 106.243 1 Template:Sidebar_with_collapsible_lists"," 5.63% 96.711 1 Template:Short_description"," 5.42% 93.030 14 Template:Cite_arXiv"," 5.18% 88.978 12 Template:Cite_conference"," 3.96% 68.020 2 Template:Pagetype"]},"scribunto":{"limitreport-timeusage":{"value":"1.156","limit":"10.000"},"limitreport-memusage":{"value":9231749,"limit":52428800},"limitreport-logs":"1 1 Sepp Hochreiter\n2 2 Jürgen Schmidhuber\n","limitreport-profile":[["?","240","20.3"],["MediaWiki\\Extension\\Scribunto\\Engines\\LuaSandbox\\LuaSandboxCallback::callParserFunction","200","16.9"],["recursiveClone \u003CmwInit.lua:45\u003E","100","8.5"],["MediaWiki\\Extension\\Scribunto\\Engines\\LuaSandbox\\LuaSandboxCallback::gsub","100","8.5"],["dataWrapper \u003Cmw.lua:672\u003E","80","6.8"],["MediaWiki\\Extension\\Scribunto\\Engines\\LuaSandbox\\LuaSandboxCallback::getAllExpandedArguments","40","3.4"],["\u003Cmw.title.lua:50\u003E","40","3.4"],["tostring","40","3.4"],["MediaWiki\\Extension\\Scribunto\\Engines\\LuaSandbox\\LuaSandboxCallback::unstripNoWiki","40","3.4"],["type","40","3.4"],["[others]","260","22.0"]]},"cachereport":{"origin":"mw-api-int.codfw.main-6fdd9f9b88-7l52t","timestamp":"20241129131521","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Recurrent neural network","url":"https:\/\/en.wikipedia.org\/wiki\/Recurrent_neural_network","sameAs":"http:\/\/www.wikidata.org\/entity\/Q1457734","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q1457734","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2005-04-07T20:49:29Z","dateModified":"2024-11-17T07:27:18Z","headline":"class of artificial neural network where connections between units form a directed graph along a temporal sequence"}</script> </body> </html>