CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Autoencoder - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"8a7c10bb-ea54-45c9-a503-07392243179c","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Autoencoder","wgTitle":"Autoencoder","wgCurRevisionId":1275135656,"wgRevisionId":1275135656,"wgArticleId":6836612,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Articles with short description","Short description matches Wikidata","Use dmy dates from March 2020","Neural network architectures","Unsupervised learning","Dimension reduction"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Autoencoder","wgRelevantArticleId":6836612,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":50000,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q786435","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false}; RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.cite.styles":"ready","ext.math.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","mediawiki.page.media","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.20"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/3/37/Autoencoder_schema.png"> <meta property="og:image:width" content="1200"> <meta property="og:image:height" content="1092"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/3/37/Autoencoder_schema.png/800px-Autoencoder_schema.png"> <meta property="og:image:width" content="800"> <meta property="og:image:height" content="728"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/3/37/Autoencoder_schema.png/640px-Autoencoder_schema.png"> <meta property="og:image:width" content="640"> <meta property="og:image:height" content="582"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Autoencoder - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Autoencoder"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Autoencoder&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Autoencoder"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Autoencoder rootpage-Autoencoder skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" title="Main menu" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia"><span>Contents</span></a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events"><span>Current events</span></a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x"><span>Random article</span></a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works"><span>About Wikipedia</span></a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia"><span>Contact us</span></a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia"><span>Help</span></a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia"><span>Learn to edit</span></a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors"><span>Community portal</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia"><span>Upload file</span></a></li><li id="n-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages"><span>Special pages</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=en.wikipedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Autoencoder" title="You are encouraged to create an account and log in; however, it is not mandatory" class=""><span>Create account</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Autoencoder" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=en.wikipedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Autoencoder" title="You are encouraged to create an account and log in; however, it is not mandatory"><span class="vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd"></span> <span>Create account</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Autoencoder" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing"><span>learn more</span></a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y"><span>Contributions</span></a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n"><span>Talk</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-Mathematical_principles" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Mathematical_principles"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Mathematical principles</span> </div> </a> <button aria-controls="toc-Mathematical_principles-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Mathematical principles subsection</span> </button> <ul id="toc-Mathematical_principles-sublist" class="vector-toc-list"> <li id="toc-Definition" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Definition"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.1</span> <span>Definition</span> </div> </a> <ul id="toc-Definition-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Training_an_autoencoder" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Training_an_autoencoder"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.2</span> <span>Training an autoencoder</span> </div> </a> <ul id="toc-Training_an_autoencoder-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Interpretation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Interpretation"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.3</span> <span>Interpretation</span> </div> </a> <ul id="toc-Interpretation-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Variations" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Variations"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Variations</span> </div> </a> <button aria-controls="toc-Variations-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Variations subsection</span> </button> <ul id="toc-Variations-sublist" class="vector-toc-list"> <li id="toc-Variational_autoencoder_(VAE)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Variational_autoencoder_(VAE)"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1</span> <span>Variational autoencoder (VAE)</span> </div> </a> <ul id="toc-Variational_autoencoder_(VAE)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Sparse_autoencoder_(SAE)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Sparse_autoencoder_(SAE)"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.2</span> <span>Sparse autoencoder (SAE)</span> </div> </a> <ul id="toc-Sparse_autoencoder_(SAE)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Denoising_autoencoder_(DAE)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Denoising_autoencoder_(DAE)"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.3</span> <span>Denoising autoencoder (DAE)</span> </div> </a> <ul id="toc-Denoising_autoencoder_(DAE)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Contractive_autoencoder_(CAE)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Contractive_autoencoder_(CAE)"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.4</span> <span>Contractive autoencoder (CAE)</span> </div> </a> <ul id="toc-Contractive_autoencoder_(CAE)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Minimum_description_length_autoencoder_(MDL-AE)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Minimum_description_length_autoencoder_(MDL-AE)"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.5</span> <span>Minimum description length autoencoder (MDL-AE)</span> </div> </a> <ul id="toc-Minimum_description_length_autoencoder_(MDL-AE)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Concrete_autoencoder_(CAE)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Concrete_autoencoder_(CAE)"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.6</span> <span>Concrete autoencoder (CAE)</span> </div> </a> <ul id="toc-Concrete_autoencoder_(CAE)-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Advantages_of_depth" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Advantages_of_depth"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Advantages of depth</span> </div> </a> <button aria-controls="toc-Advantages_of_depth-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Advantages of depth subsection</span> </button> <ul id="toc-Advantages_of_depth-sublist" class="vector-toc-list"> <li id="toc-Training" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Training"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Training</span> </div> </a> <ul id="toc-Training-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-History" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#History"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>History</span> </div> </a> <ul id="toc-History-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Applications" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Applications"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Applications</span> </div> </a> <button aria-controls="toc-Applications-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Applications subsection</span> </button> <ul id="toc-Applications-sublist" class="vector-toc-list"> <li id="toc-Dimensionality_reduction" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Dimensionality_reduction"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.1</span> <span>Dimensionality reduction</span> </div> </a> <ul id="toc-Dimensionality_reduction-sublist" class="vector-toc-list"> <li id="toc-Principal_component_analysis" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Principal_component_analysis"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.1.1</span> <span>Principal component analysis</span> </div> </a> <ul id="toc-Principal_component_analysis-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Information_retrieval_and_Search_engine_optimization" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Information_retrieval_and_Search_engine_optimization"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.2</span> <span>Information retrieval and Search engine optimization</span> </div> </a> <ul id="toc-Information_retrieval_and_Search_engine_optimization-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Anomaly_detection" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Anomaly_detection"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.3</span> <span>Anomaly detection</span> </div> </a> <ul id="toc-Anomaly_detection-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Image_processing" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Image_processing"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.4</span> <span>Image processing</span> </div> </a> <ul id="toc-Image_processing-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Drug_discovery" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Drug_discovery"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.5</span> <span>Drug discovery</span> </div> </a> <ul id="toc-Drug_discovery-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Popularity_prediction" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Popularity_prediction"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.6</span> <span>Popularity prediction</span> </div> </a> <ul id="toc-Popularity_prediction-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Machine_translation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Machine_translation"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.7</span> <span>Machine translation</span> </div> </a> <ul id="toc-Machine_translation-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Communication_Systems" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Communication_Systems"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.8</span> <span>Communication Systems</span> </div> </a> <ul id="toc-Communication_Systems-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>See also</span> </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Further_reading" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Further_reading"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>Further reading</span> </div> </a> <ul id="toc-Further_reading-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" title="Table of Contents" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Autoencoder</span></h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 20 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-20" aria-hidden="true" ><span class="vector-icon mw-ui-icon-language-progressive mw-ui-icon-wikimedia-language-progressive"></span> <span class="vector-dropdown-label-text">20 languages</span> </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D9%85%D8%B1%D9%85%D8%B2_%D8%AA%D9%84%D9%82%D8%A7%D8%A6%D9%8A" title="مرمز تلقائي – Arabic" lang="ar" hreflang="ar" data-title="مرمز تلقائي" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target"><span>العربية</span></a></li><li class="interlanguage-link interwiki-ca mw-list-item"><a href="https://ca.wikipedia.org/wiki/Autoencoder" title="Autoencoder – Catalan" lang="ca" hreflang="ca" data-title="Autoencoder" data-language-autonym="Català" data-language-local-name="Catalan" class="interlanguage-link-target"><span>Català</span></a></li><li class="interlanguage-link interwiki-cs mw-list-item"><a href="https://cs.wikipedia.org/wiki/Autoenkod%C3%A9r" title="Autoenkodér – Czech" lang="cs" hreflang="cs" data-title="Autoenkodér" data-language-autonym="Čeština" data-language-local-name="Czech" class="interlanguage-link-target"><span>Čeština</span></a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Autoencoder" title="Autoencoder – German" lang="de" hreflang="de" data-title="Autoencoder" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target"><span>Deutsch</span></a></li><li class="interlanguage-link interwiki-el mw-list-item"><a href="https://el.wikipedia.org/wiki/Autoencoder" title="Autoencoder – Greek" lang="el" hreflang="el" data-title="Autoencoder" data-language-autonym="Ελληνικά" data-language-local-name="Greek" class="interlanguage-link-target"><span>Ελληνικά</span></a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Autocodificador" title="Autocodificador – Spanish" lang="es" hreflang="es" data-title="Autocodificador" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target"><span>Español</span></a></li><li class="interlanguage-link interwiki-fa mw-list-item"><a href="https://fa.wikipedia.org/wiki/%D8%AE%D9%88%D8%AF%D8%B1%D9%85%D8%B2%DA%AF%D8%B0%D8%A7%D8%B1" title="خودرمزگذار – Persian" lang="fa" hreflang="fa" data-title="خودرمزگذار" data-language-autonym="فارسی" data-language-local-name="Persian" class="interlanguage-link-target"><span>فارسی</span></a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/Auto-encodeur" title="Auto-encodeur – French" lang="fr" hreflang="fr" data-title="Auto-encodeur" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target"><span>Français</span></a></li><li class="interlanguage-link interwiki-ko mw-list-item"><a href="https://ko.wikipedia.org/wiki/%EC%98%A4%ED%86%A0%EC%9D%B8%EC%BD%94%EB%8D%94" title="오토인코더 – Korean" lang="ko" hreflang="ko" data-title="오토인코더" data-language-autonym="한국어" data-language-local-name="Korean" class="interlanguage-link-target"><span>한국어</span></a></li><li class="interlanguage-link interwiki-hy mw-list-item"><a href="https://hy.wikipedia.org/wiki/%D4%BB%D5%B6%D6%84%D5%B6%D5%A1%D5%AF%D5%B8%D5%A4%D5%A1%D5%BE%D5%B8%D6%80%D5%AB%D5%B9" title="Ինքնակոդավորիչ – Armenian" lang="hy" hreflang="hy" data-title="Ինքնակոդավորիչ" data-language-autonym="Հայերեն" data-language-local-name="Armenian" class="interlanguage-link-target"><span>Հայերեն</span></a></li><li class="interlanguage-link interwiki-id mw-list-item"><a href="https://id.wikipedia.org/wiki/Autopenyandi" title="Autopenyandi – Indonesian" lang="id" hreflang="id" data-title="Autopenyandi" data-language-autonym="Bahasa Indonesia" data-language-local-name="Indonesian" class="interlanguage-link-target"><span>Bahasa Indonesia</span></a></li><li class="interlanguage-link interwiki-it mw-list-item"><a href="https://it.wikipedia.org/wiki/Autocodificatore" title="Autocodificatore – Italian" lang="it" hreflang="it" data-title="Autocodificatore" data-language-autonym="Italiano" data-language-local-name="Italian" class="interlanguage-link-target"><span>Italiano</span></a></li><li class="interlanguage-link interwiki-ja mw-list-item"><a href="https://ja.wikipedia.org/wiki/%E3%82%AA%E3%83%BC%E3%83%88%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%80" title="オートエンコーダ – Japanese" lang="ja" hreflang="ja" data-title="オートエンコーダ" data-language-autonym="日本語" data-language-local-name="Japanese" class="interlanguage-link-target"><span>日本語</span></a></li><li class="interlanguage-link interwiki-ru mw-list-item"><a href="https://ru.wikipedia.org/wiki/%D0%90%D0%B2%D1%82%D0%BE%D0%BA%D0%BE%D0%B4%D0%B8%D1%80%D0%BE%D0%B2%D1%89%D0%B8%D0%BA" title="Автокодировщик – Russian" lang="ru" hreflang="ru" data-title="Автокодировщик" data-language-autonym="Русский" data-language-local-name="Russian" class="interlanguage-link-target"><span>Русский</span></a></li><li class="interlanguage-link interwiki-th mw-list-item"><a href="https://th.wikipedia.org/wiki/%E0%B8%95%E0%B8%B1%E0%B8%A7%E0%B9%80%E0%B8%82%E0%B9%89%E0%B8%B2%E0%B8%A3%E0%B8%AB%E0%B8%B1%E0%B8%AA%E0%B8%AD%E0%B8%B1%E0%B8%95%E0%B9%82%E0%B8%99%E0%B8%A1%E0%B8%B1%E0%B8%95%E0%B8%B4" title="ตัวเข้ารหัสอัตโนมัติ – Thai" lang="th" hreflang="th" data-title="ตัวเข้ารหัสอัตโนมัติ" data-language-autonym="ไทย" data-language-local-name="Thai" class="interlanguage-link-target"><span>ไทย</span></a></li><li class="interlanguage-link interwiki-tr mw-list-item"><a href="https://tr.wikipedia.org/wiki/Otokodlay%C4%B1c%C4%B1" title="Otokodlayıcı – Turkish" lang="tr" hreflang="tr" data-title="Otokodlayıcı" data-language-autonym="Türkçe" data-language-local-name="Turkish" class="interlanguage-link-target"><span>Türkçe</span></a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%90%D0%B2%D1%82%D0%BE%D0%BA%D0%BE%D0%B4%D1%83%D0%B2%D0%B0%D0%BB%D1%8C%D0%BD%D0%B8%D0%BA" title="Автокодувальник – Ukrainian" lang="uk" hreflang="uk" data-title="Автокодувальник" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target"><span>Українська</span></a></li><li class="interlanguage-link interwiki-vi mw-list-item"><a href="https://vi.wikipedia.org/wiki/B%E1%BB%99_t%E1%BB%B1_m%C3%A3_h%C3%B3a" title="Bộ tự mã hóa – Vietnamese" lang="vi" hreflang="vi" data-title="Bộ tự mã hóa" data-language-autonym="Tiếng Việt" data-language-local-name="Vietnamese" class="interlanguage-link-target"><span>Tiếng Việt</span></a></li><li class="interlanguage-link interwiki-zh-yue mw-list-item"><a href="https://zh-yue.wikipedia.org/wiki/%E8%87%AA%E7%B7%A8%E7%A2%BC%E5%99%A8" title="自編碼器 – Cantonese" lang="yue" hreflang="yue" data-title="自編碼器" data-language-autonym="粵語" data-language-local-name="Cantonese" class="interlanguage-link-target"><span>粵語</span></a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E8%87%AA%E7%BC%96%E7%A0%81%E5%99%A8" title="自编码器 – Chinese" lang="zh" hreflang="zh" data-title="自编码器" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target"><span>中文</span></a></li> </ul> <div class="after-portlet after-portlet-lang"><span class="wb-langlinks-edit wb-langlinks-link"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q786435#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></span></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Autoencoder" title="View the content page [c]" accesskey="c"><span>Article</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Autoencoder" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t"><span>Talk</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Autoencoder"><span>Read</span></a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Autoencoder&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Autoencoder&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Autoencoder"><span>Read</span></a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Autoencoder&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Autoencoder&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Autoencoder" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Autoencoder" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-upload" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u"><span>Upload file</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Autoencoder&oldid=1275135656" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Autoencoder&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Autoencoder&id=1275135656&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAutoencoder"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAutoencoder"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Autoencoder&action=show-download-screen" title="Download this page as a PDF file"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Autoencoder&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q786435" title="Structured data on this page hosted by Wikidata [g]" accesskey="g"><span>Wikidata item</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Neural network that learns efficient data encoding in an unsupervised manner</div> <style data-mw-deduplicate="TemplateStyles:r1236090951">.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}</style><div role="note" class="hatnote navigation-not-searchable">Not to be confused with <a href="/wiki/Autocoder" title="Autocoder">Autocoder</a> or <a href="/wiki/Autocode" title="Autocode">Autocode</a>.</div> <p class="mw-empty-elt"> </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Autoencoder_schema.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/3/37/Autoencoder_schema.png/250px-Autoencoder_schema.png" decoding="async" width="250" height="227" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/37/Autoencoder_schema.png/375px-Autoencoder_schema.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/37/Autoencoder_schema.png/500px-Autoencoder_schema.png 2x" data-file-width="841" data-file-height="765" /></a><figcaption>A schema of an <i>autoencoder</i>. An autoencoder has two main parts: an <i>encoder</i> that maps the message to a code, and a <i>decoder</i> that reconstructs the message from the code.</figcaption></figure> <style data-mw-deduplicate="TemplateStyles:r1244144826">.mw-parser-output .machine-learning-list-title{background-color:#ddddff}html.skin-theme-clientpref-night .mw-parser-output .machine-learning-list-title{background-color:#222}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .machine-learning-list-title{background-color:#222}}</style> <style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1246091330">.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}</style><style data-mw-deduplicate="TemplateStyles:r886047488">.mw-parser-output .nobold{font-weight:normal}</style><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r886047488" /><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="/wiki/Machine_learning" title="Machine learning">Machine learning</a><br />and <a href="/wiki/Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Paradigms</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Supervised_learning" title="Supervised learning">Supervised learning</a></li> <li><a href="/wiki/Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li> <li><a href="/wiki/Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li> <li><a href="/wiki/Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li> <li><a href="/wiki/Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li> <li><a href="/wiki/Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li> <li><a href="/wiki/Online_machine_learning" title="Online machine learning">Online learning</a></li> <li><a href="/wiki/Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li> <li><a href="/wiki/Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li> <li><a href="/wiki/Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li> <li><a href="/wiki/Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li> <li><a href="/wiki/Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li> <li><a href="/wiki/Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Problems</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Statistical_classification" title="Statistical classification">Classification</a></li> <li><a href="/wiki/Generative_model" title="Generative model">Generative modeling</a></li> <li><a href="/wiki/Regression_analysis" title="Regression analysis">Regression</a></li> <li><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></li> <li><a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li> <li><a href="/wiki/Density_estimation" title="Density estimation">Density estimation</a></li> <li><a href="/wiki/Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li> <li><a href="/wiki/Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li> <li><a href="/wiki/Automated_machine_learning" title="Automated machine learning">AutoML</a></li> <li><a href="/wiki/Association_rule_learning" title="Association rule learning">Association rules</a></li> <li><a href="/wiki/Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li> <li><a href="/wiki/Structured_prediction" title="Structured prediction">Structured prediction</a></li> <li><a href="/wiki/Feature_engineering" title="Feature engineering">Feature engineering</a></li> <li><a href="/wiki/Feature_learning" title="Feature learning">Feature learning</a></li> <li><a href="/wiki/Learning_to_rank" title="Learning to rank">Learning to rank</a></li> <li><a href="/wiki/Grammar_induction" title="Grammar induction">Grammar induction</a></li> <li><a href="/wiki/Ontology_learning" title="Ontology learning">Ontology learning</a></li> <li><a href="/wiki/Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="/wiki/Supervised_learning" title="Supervised learning">Supervised learning</a><br /><span class="nobold"><span style="font-size:85%;">(<b><a href="/wiki/Statistical_classification" title="Statistical classification">classification</a></b> • <b><a href="/wiki/Regression_analysis" title="Regression analysis">regression</a></b>)</span></span> </div></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li> <li><a href="/wiki/Decision_tree_learning" title="Decision tree learning">Decision trees</a></li> <li><a href="/wiki/Ensemble_learning" title="Ensemble learning">Ensembles</a> <ul><li><a href="/wiki/Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li> <li><a href="/wiki/Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li> <li><a href="/wiki/Random_forest" title="Random forest">Random forest</a></li></ul></li> <li><a href="/wiki/K-nearest_neighbors_algorithm" title="K-nearest neighbors algorithm"><i>k</i>-NN</a></li> <li><a href="/wiki/Linear_regression" title="Linear regression">Linear regression</a></li> <li><a href="/wiki/Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li> <li><a href="/wiki/Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li> <li><a href="/wiki/Logistic_regression" title="Logistic regression">Logistic regression</a></li> <li><a href="/wiki/Perceptron" title="Perceptron">Perceptron</a></li> <li><a href="/wiki/Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li> <li><a href="/wiki/Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/BIRCH" title="BIRCH">BIRCH</a></li> <li><a href="/wiki/CURE_algorithm" title="CURE algorithm">CURE</a></li> <li><a href="/wiki/Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li> <li><a href="/wiki/K-means_clustering" title="K-means clustering"><i>k</i>-means</a></li> <li><a href="/wiki/Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li> <li><a href="/wiki/Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li> <li><br /><a href="/wiki/DBSCAN" title="DBSCAN">DBSCAN</a></li> <li><a href="/wiki/OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li> <li><a href="/wiki/Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Factor_analysis" title="Factor analysis">Factor analysis</a></li> <li><a href="/wiki/Canonical_correlation" title="Canonical correlation">CCA</a></li> <li><a href="/wiki/Independent_component_analysis" title="Independent component analysis">ICA</a></li> <li><a href="/wiki/Linear_discriminant_analysis" title="Linear discriminant analysis">LDA</a></li> <li><a href="/wiki/Non-negative_matrix_factorization" title="Non-negative matrix factorization">NMF</a></li> <li><a href="/wiki/Principal_component_analysis" title="Principal component analysis">PCA</a></li> <li><a href="/wiki/Proper_generalized_decomposition" title="Proper generalized decomposition">PGD</a></li> <li><a href="/wiki/T-distributed_stochastic_neighbor_embedding" title="T-distributed stochastic neighbor embedding">t-SNE</a></li> <li><a href="/wiki/Sparse_dictionary_learning" title="Sparse dictionary learning">SDL</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Graphical_model" title="Graphical model">Graphical models</a> <ul><li><a href="/wiki/Bayesian_network" title="Bayesian network">Bayes net</a></li> <li><a href="/wiki/Conditional_random_field" title="Conditional random field">Conditional random field</a></li> <li><a href="/wiki/Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Random_sample_consensus" title="Random sample consensus">RANSAC</a></li> <li><a href="/wiki/K-nearest_neighbors_algorithm" title="K-nearest neighbors algorithm"><i>k</i>-NN</a></li> <li><a href="/wiki/Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li> <li><a href="/wiki/Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural network</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a class="mw-selflink selflink">Autoencoder</a></li> <li><a href="/wiki/Deep_learning" title="Deep learning">Deep learning</a></li> <li><a href="/wiki/Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li> <li><a href="/wiki/Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network</a> <ul><li><a href="/wiki/Long_short-term_memory" title="Long short-term memory">LSTM</a></li> <li><a href="/wiki/Gated_recurrent_unit" title="Gated recurrent unit">GRU</a></li> <li><a href="/wiki/Echo_state_network" title="Echo state network">ESN</a></li> <li><a href="/wiki/Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li> <li><a href="/wiki/Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a> <ul><li><a href="/wiki/Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li> <li><a href="/wiki/Generative_adversarial_network" title="Generative adversarial network">GAN</a></li> <li><a href="/wiki/Diffusion_model" title="Diffusion model">Diffusion model</a></li> <li><a href="/wiki/Self-organizing_map" title="Self-organizing map">SOM</a></li> <li><a href="/wiki/Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a> <ul><li><a href="/wiki/U-Net" title="U-Net">U-Net</a></li> <li><a href="/wiki/LeNet" title="LeNet">LeNet</a></li> <li><a href="/wiki/AlexNet" title="AlexNet">AlexNet</a></li> <li><a href="/wiki/DeepDream" title="DeepDream">DeepDream</a></li></ul></li> <li><a href="/wiki/Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li> <li><a href="/wiki/Transformer_(machine_learning_model)" class="mw-redirect" title="Transformer (machine learning model)">Transformer</a> <ul><li><a href="/wiki/Vision_transformer" title="Vision transformer">Vision</a></li></ul></li> <li><a href="/wiki/Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li> <li><a href="/wiki/Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li> <li><a href="/wiki/Memtransistor" title="Memtransistor">Memtransistor</a></li> <li><a href="/wiki/Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="/wiki/Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Q-learning" title="Q-learning">Q-learning</a></li> <li><a href="/wiki/State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li> <li><a href="/wiki/Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li> <li><a href="/wiki/Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a> <ul><li><a href="/wiki/Self-play_(reinforcement_learning_technique)" class="mw-redirect" title="Self-play (reinforcement learning technique)">Self-play</a></li></ul></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Learning with humans</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li> <li><a href="/wiki/Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li> <li><a href="/wiki/Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li> <li><a href="/wiki/Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">RLHF</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Model diagnostics</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li> <li><a href="/wiki/Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li> <li><a href="/wiki/Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li> <li><a href="/wiki/Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li> <li><a href="/wiki/Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li> <li><a href="/wiki/Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li> <li><a href="/wiki/Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li> <li><a href="/wiki/Occam_learning" title="Occam learning">Occam learning</a></li> <li><a href="/wiki/Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li> <li><a href="/wiki/Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li> <li><a href="/wiki/Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li> <li><a href="/wiki/Topological_deep_learning" title="Topological deep learning">Topological deep learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/ECML_PKDD" title="ECML PKDD">ECML PKDD</a></li> <li><a href="/wiki/Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li> <li><a href="/wiki/International_Conference_on_Machine_Learning" title="International Conference on Machine Learning">ICML</a></li> <li><a href="/wiki/International_Conference_on_Learning_Representations" title="International Conference on Learning Representations">ICLR</a></li> <li><a href="/wiki/International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li> <li><a href="/wiki/Machine_Learning_(journal)" title="Machine Learning (journal)">ML</a></li> <li><a href="/wiki/Journal_of_Machine_Learning_Research" title="Journal of Machine Learning Research">JMLR</a></li></ul></div></div></td> </tr><tr><td class="sidebar-content"> <div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Related articles</div><div class="sidebar-list-content mw-collapsible-content hlist"> <ul><li><a href="/wiki/Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li> <li><a href="/wiki/List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a> <ul><li><a href="/wiki/List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li> <li><a href="/wiki/Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td> </tr><tr><td class="sidebar-navbar"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Machine_learning" title="Template:Machine learning"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Machine_learning" title="Template talk:Machine learning"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Machine_learning" title="Special:EditPage/Template:Machine learning"><abbr title="Edit this template">e</abbr></a></li></ul></div></td></tr></tbody></table> <p>An <b>autoencoder</b> is a type of <a href="/wiki/Artificial_neural_network" class="mw-redirect" title="Artificial neural network">artificial neural network</a> used to learn <a href="/wiki/Feature_learning" title="Feature learning">efficient codings</a> of unlabeled data (<a href="/wiki/Unsupervised_learning" title="Unsupervised learning">unsupervised learning</a>). An autoencoder learns two functions: an encoding function that transforms the input data, and a decoding function that recreates the input data from the encoded representation. The autoencoder learns an efficient representation (encoding) for a set of data, typically for <a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">dimensionality reduction</a>, to generate lower-dimensional embeddings for subsequent use by other <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a> algorithms.<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> </p><p>Variants exist which aim to make the learned representations assume useful properties.<sup id="cite_ref-:0_2-0" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> Examples are regularized autoencoders (<i>sparse</i>, <i>denoising</i> and <i>contractive</i> autoencoders), which are effective in learning representations for subsequent <a href="/wiki/Statistical_classification" title="Statistical classification">classification</a> tasks,<sup id="cite_ref-:4_3-0" class="reference"><a href="#cite_note-:4-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> and <a href="/wiki/Variational_autoencoder" title="Variational autoencoder"><i>variational</i> autoencoders</a>, which can be used as <a href="/wiki/Generative_model" title="Generative model">generative models</a>.<sup id="cite_ref-:11_4-0" class="reference"><a href="#cite_note-:11-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup> Autoencoders are applied to many problems, including <a href="/wiki/Facial_recognition_system" title="Facial recognition system">facial recognition</a>,<sup id="cite_ref-5" class="reference"><a href="#cite_note-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> <a href="/wiki/Feature_(computer_vision)" title="Feature (computer vision)">feature detection</a>,<sup id="cite_ref-:2_6-0" class="reference"><a href="#cite_note-:2-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> <a href="/wiki/Anomaly_detection" title="Anomaly detection">anomaly detection</a>, and <a href="/wiki/Word_embedding" title="Word embedding">learning the meaning of words</a>.<sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> In terms of <a href="/wiki/Synthetic_data" title="Synthetic data">data synthesis</a>, autoencoders can also be used to randomly generate new data that is similar to the input (training) data.<sup id="cite_ref-:2_6-1" class="reference"><a href="#cite_note-:2-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> </p> <style data-mw-deduplicate="TemplateStyles:r886046785">.mw-parser-output .toclimit-2 .toclevel-1 ul,.mw-parser-output .toclimit-3 .toclevel-2 ul,.mw-parser-output .toclimit-4 .toclevel-3 ul,.mw-parser-output .toclimit-5 .toclevel-4 ul,.mw-parser-output .toclimit-6 .toclevel-5 ul,.mw-parser-output .toclimit-7 .toclevel-6 ul{display:none}</style><div class="toclimit-3"><meta property="mw:PageProp/toc" /></div> <div class="mw-heading mw-heading2"><h2 id="Mathematical_principles">Mathematical principles</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=1" title="Edit section: Mathematical principles"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Definition">Definition</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=2" title="Edit section: Definition"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div><p> An autoencoder is defined by the following components: </p><blockquote><p>Two sets: the space of decoded messages <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8c7e5461c5286852df4ef652fca7e4b0b63030e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.875ex; height:2.176ex;" alt="{\displaystyle {\mathcal {X}}}" /></span>; the space of encoded messages <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {Z}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {Z}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e9be1e2c3c82e7b8055fe26eb4cf2caac6f8ec73" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.783ex; height:2.176ex;" alt="{\displaystyle {\mathcal {Z}}}" /></span>. Typically <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8c7e5461c5286852df4ef652fca7e4b0b63030e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.875ex; height:2.176ex;" alt="{\displaystyle {\mathcal {X}}}" /></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {Z}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {Z}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e9be1e2c3c82e7b8055fe26eb4cf2caac6f8ec73" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.783ex; height:2.176ex;" alt="{\displaystyle {\mathcal {Z}}}" /></span> are <a href="/wiki/Euclidean_space" title="Euclidean space">Euclidean spaces</a>, that is, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}=\mathbb {R} ^{m},{\mathcal {Z}}=\mathbb {R} ^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>=</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>m</mi> </mrow> </msup> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> <mo>=</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}=\mathbb {R} ^{m},{\mathcal {Z}}=\mathbb {R} ^{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0019b12cb0c515f608bdec62ff3e31248ce282f0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:17.139ex; height:2.676ex;" alt="{\displaystyle {\mathcal {X}}=\mathbb {R} ^{m},{\mathcal {Z}}=\mathbb {R} ^{n}}" /></span> with <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m>n.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>m</mi> <mo>></mo> <mi>n</mi> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle m>n.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/982733f442459689d943182858c50d3942aaa764" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.18ex; height:1.843ex;" alt="{\displaystyle m>n.}" /></span> </p></blockquote><blockquote><p>Two <a href="/wiki/Parameterization" class="mw-redirect" title="Parameterization">parametrized</a> families of functions: the encoder family <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E_{\phi }:{\mathcal {X}}\rightarrow {\mathcal {Z}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo>:</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo stretchy="false">→</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle E_{\phi }:{\mathcal {X}}\rightarrow {\mathcal {Z}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1ca207ad612d4f68f449647ef438f20fc6f1dfd8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:12.136ex; height:2.843ex;" alt="{\displaystyle E_{\phi }:{\mathcal {X}}\rightarrow {\mathcal {Z}}}" /></span>, parametrized by <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>ϕ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \phi }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/72b1f30316670aee6270a28334bdf4f5072cdde4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" /></span>; the decoder family <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D_{\theta }:{\mathcal {Z}}\rightarrow {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo>:</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> <mo stretchy="false">→</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D_{\theta }:{\mathcal {Z}}\rightarrow {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f67a834ca4cd55d93088f2ca566739c100500f75" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:12.137ex; height:2.509ex;" alt="{\displaystyle D_{\theta }:{\mathcal {Z}}\rightarrow {\mathcal {X}}}" /></span>, parametrized by <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>θ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \theta }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6e5ab2664b422d53eb0c7df3b87e1360d75ad9af" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.09ex; height:2.176ex;" alt="{\displaystyle \theta }" /></span>.</p></blockquote><p>For any <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x\in {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x\in {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0b3cd596d560b9f3bc71615911c2438cbbf49c15" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.046ex; height:2.176ex;" alt="{\displaystyle x\in {\mathcal {X}}}" /></span>, we usually write <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z=E_{\phi }(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>z</mi> <mo>=</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle z=E_{\phi }(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/002c12f50476471d3e541bdd77f7a522cbd80444" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:10.253ex; height:3.009ex;" alt="{\displaystyle z=E_{\phi }(x)}" /></span>, and refer to it as the code, the <a href="/wiki/Latent_variable" class="mw-redirect" title="Latent variable">latent variable</a>, latent representation, latent vector, etc. Conversely, for any <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z\in {\mathcal {Z}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>z</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle z\in {\mathcal {Z}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3fd7a639b0e1c2ba8d6825ca79b144c906980979" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.711ex; height:2.176ex;" alt="{\displaystyle z\in {\mathcal {Z}}}" /></span>, we usually write <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x'=D_{\theta }(z)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo>=</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>z</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x'=D_{\theta }(z)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2959b5a51a87eac650c51768411b6a310c197b67" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:10.938ex; height:3.009ex;" alt="{\displaystyle x'=D_{\theta }(z)}" /></span>, and refer to it as the (decoded) message. </p><p>Usually, both the encoder and the decoder are defined as <a href="/wiki/Multilayer_perceptron" title="Multilayer perceptron">multilayer perceptrons</a> (MLPs). For example, a one-layer-MLP encoder <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E_{\phi }}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle E_{\phi }}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c08f0cada78242b00f566df96531e52fc6277fb6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.927ex; height:2.843ex;" alt="{\displaystyle E_{\phi }}" /></span> is: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E_{\phi }(\mathbf {x} )=\sigma (Wx+b)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="bold">x</mi> </mrow> <mo stretchy="false">)</mo> <mo>=</mo> <mi>σ</mi> <mo stretchy="false">(</mo> <mi>W</mi> <mi>x</mi> <mo>+</mo> <mi>b</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle E_{\phi }(\mathbf {x} )=\sigma (Wx+b)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8ce6546eb990e3b456b461f905fcf795affbddd9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:19.988ex; height:3.009ex;" alt="{\displaystyle E_{\phi }(\mathbf {x} )=\sigma (Wx+b)}" /></span></dd></dl> <p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>σ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \sigma }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/59f59b7c3e6fdb1d0365a494b81fb9a696138c36" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle \sigma }" /></span> is an element-wise <a href="/wiki/Activation_function" title="Activation function">activation function</a>, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>W</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle W}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/54a9c4c547f4d6111f81946cad242b18298d70b7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.176ex;" alt="{\displaystyle W}" /></span> is a "weight" matrix, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>b</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle b}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f11423fbb2e967f986e36804a8ae4271734917c3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.998ex; height:2.176ex;" alt="{\displaystyle b}" /></span> is a "bias" vector. </p> <div class="mw-heading mw-heading3"><h3 id="Training_an_autoencoder">Training an autoencoder</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=3" title="Edit section: Training an autoencoder"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>An autoencoder, by itself, is simply a tuple of two functions. To judge its <i>quality</i>, we need a <i>task</i>. A task is defined by a reference probability distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{ref}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mu _{ref}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/15d15b590538ca3450d814c51e7707812a8c26b4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.046ex; height:2.343ex;" alt="{\displaystyle \mu _{ref}}" /></span> over <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8c7e5461c5286852df4ef652fca7e4b0b63030e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.875ex; height:2.176ex;" alt="{\displaystyle {\mathcal {X}}}" /></span>, and a "reconstruction quality" function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle d:{\mathcal {X}}\times {\mathcal {X}}\to [0,\infty ]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>d</mi> <mo>:</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>×</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo stretchy="false">→</mo> <mo stretchy="false">[</mo> <mn>0</mn> <mo>,</mo> <mi mathvariant="normal">∞</mi> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle d:{\mathcal {X}}\times {\mathcal {X}}\to [0,\infty ]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e69538969fb3dc0706fb3368e6312ccb95cf2d6e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.172ex; height:2.843ex;" alt="{\displaystyle d:{\mathcal {X}}\times {\mathcal {X}}\to [0,\infty ]}" /></span>, such that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle d(x,x')}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>d</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle d(x,x')}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c46ecfdcff1eb6b830f767255959144e33e680a3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.403ex; height:3.009ex;" alt="{\displaystyle d(x,x')}" /></span> measures how much <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x'}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>x</mi> <mo>′</mo> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x'}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0ac74959896052e160a5953102e4bc3850fe93b2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.014ex; height:2.509ex;" alt="{\displaystyle x'}" /></span> differs from <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span>. </p><p>With those, we can define the loss function for the autoencoder as<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L(\theta ,\phi ):=\mathbb {\mathbb {E} } _{x\sim \mu _{ref}}[d(x,D_{\theta }(E_{\phi }(x)))]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>:=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∼</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> </mrow> </msub> <mo stretchy="false">[</mo> <mi>d</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L(\theta ,\phi ):=\mathbb {\mathbb {E} } _{x\sim \mu _{ref}}[d(x,D_{\theta }(E_{\phi }(x)))]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b67d5c0347ed7cd5dab78196fe3501a843f97d34" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:35.247ex; height:3.343ex;" alt="{\displaystyle L(\theta ,\phi ):=\mathbb {\mathbb {E} } _{x\sim \mu _{ref}}[d(x,D_{\theta }(E_{\phi }(x)))]}" /></span>The <i>optimal</i> autoencoder for the given task <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (\mu _{ref},d)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> <mo>,</mo> <mi>d</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (\mu _{ref},d)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4077502606b9dc074830c1abca71cc750b87fed4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:8.105ex; height:3.009ex;" alt="{\displaystyle (\mu _{ref},d)}" /></span> is then <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \arg \min _{\theta ,\phi }L(\theta ,\phi )}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>arg</mi> <mo>⁡</mo> <munder> <mo movablelimits="true" form="prefix">min</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> </mrow> </munder> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \arg \min _{\theta ,\phi }L(\theta ,\phi )}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/be5f1d9e9e86289264951861390a1649e7c99952" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:14.788ex; height:4.343ex;" alt="{\displaystyle \arg \min _{\theta ,\phi }L(\theta ,\phi )}" /></span>. The search for the optimal autoencoder can be accomplished by any mathematical optimization technique, but usually by <a href="/wiki/Gradient_descent" title="Gradient descent">gradient descent</a>. This search process is referred to as "training the autoencoder". </p><p>In most situations, the reference distribution is just the <a href="/wiki/Empirical_measure" title="Empirical measure">empirical distribution</a> given by a dataset <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{x_{1},...,x_{N}\}\subset {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">{</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </msub> <mo fence="false" stretchy="false">}</mo> <mo>⊂</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \{x_{1},...,x_{N}\}\subset {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f6afdb997fd084fba75fed0afa6e9b1658c11821" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:17.874ex; height:2.843ex;" alt="{\displaystyle \{x_{1},...,x_{N}\}\subset {\mathcal {X}}}" /></span>, so that<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{ref}={\frac {1}{N}}\sum _{i=1}^{N}\delta _{x_{i}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mi>N</mi> </mfrac> </mrow> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </munderover> <msub> <mi>δ</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mu _{ref}={\frac {1}{N}}\sum _{i=1}^{N}\delta _{x_{i}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d63652bf750ae452bed32819120001ee060b105f" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:17.003ex; height:7.343ex;" alt="{\displaystyle \mu _{ref}={\frac {1}{N}}\sum _{i=1}^{N}\delta _{x_{i}}}" /></span> </p><p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \delta _{x_{i}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>δ</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \delta _{x_{i}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/472764bb2606d7dfd874da3b4dce1075678a15d1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.83ex; height:3.009ex;" alt="{\displaystyle \delta _{x_{i}}}" /></span> is the <a href="/wiki/Dirac_measure" title="Dirac measure">Dirac measure</a>, the quality function is just L2 loss: <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle d(x,x')=\|x-x'\|_{2}^{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>d</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo stretchy="false">)</mo> <mo>=</mo> <mo fence="false" stretchy="false">‖</mo> <mi>x</mi> <mo>−</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <msubsup> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle d(x,x')=\|x-x'\|_{2}^{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/76cf47d547a92f5e5bc7938b525ce8d3fa8d7a55" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:20.065ex; height:3.176ex;" alt="{\displaystyle d(x,x')=\|x-x'\|_{2}^{2}}" /></span>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \|\cdot \|_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">‖</mo> <mo>⋅</mo> <msub> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \|\cdot \|_{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b3a8e44a2eb980f856968a6357e3d0a7c22c905f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.058ex; height:2.843ex;" alt="{\displaystyle \|\cdot \|_{2}}" /></span> is the <a href="/wiki/Norm_(mathematics)#Euclidean_norm" title="Norm (mathematics)">Euclidean norm</a>. Then the problem of searching for the optimal autoencoder is just a <a href="/wiki/Least_squares" title="Least squares">least-squares</a> optimization:<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi ),\qquad {\text{where }}L(\theta ,\phi )={\frac {1}{N}}\sum _{i=1}^{N}\|x_{i}-D_{\theta }(E_{\phi }(x_{i}))\|_{2}^{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <munder> <mo movablelimits="true" form="prefix">min</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> </mrow> </munder> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>,</mo> <mspace width="2em"></mspace> <mrow class="MJX-TeXAtom-ORD"> <mtext>where </mtext> </mrow> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mi>N</mi> </mfrac> </mrow> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> </mrow> </munderover> <mo fence="false" stretchy="false">‖</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>−</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <msubsup> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi ),\qquad {\text{where }}L(\theta ,\phi )={\frac {1}{N}}\sum _{i=1}^{N}\|x_{i}-D_{\theta }(E_{\phi }(x_{i}))\|_{2}^{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d3cb4a557857908505588af248b8abfc682e3cb5" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:60.352ex; height:7.343ex;" alt="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi ),\qquad {\text{where }}L(\theta ,\phi )={\frac {1}{N}}\sum _{i=1}^{N}\|x_{i}-D_{\theta }(E_{\phi }(x_{i}))\|_{2}^{2}}" /></span> </p> <div class="mw-heading mw-heading3"><h3 id="Interpretation">Interpretation</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=4" title="Edit section: Interpretation"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>An autoencoder has two main parts: an encoder that maps the message to a code, and a decoder that reconstructs the message from the code. An optimal autoencoder would perform as close to perfect reconstruction as possible, with "close to perfect" defined by the reconstruction quality function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle d}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>d</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle d}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e85ff03cbe0c7341af6b982e47e9f90d235c66ab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.216ex; height:2.176ex;" alt="{\displaystyle d}" /></span>. </p><p>The simplest way to perform the copying task perfectly would be to duplicate the signal. To suppress this behavior, the code space <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {Z}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {Z}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e9be1e2c3c82e7b8055fe26eb4cf2caac6f8ec73" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.783ex; height:2.176ex;" alt="{\displaystyle {\mathcal {Z}}}" /></span> usually has fewer dimensions than the message space <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8c7e5461c5286852df4ef652fca7e4b0b63030e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.875ex; height:2.176ex;" alt="{\displaystyle {\mathcal {X}}}" /></span>. </p><p>Such an autoencoder is called <i>undercomplete</i>. It can be interpreted as <a href="/wiki/Data_compression" title="Data compression">compressing</a> the message, or <a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">reducing its dimensionality</a>.<sup id="cite_ref-:12_9-0" class="reference"><a href="#cite_note-:12-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:7_10-0" class="reference"><a href="#cite_note-:7-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> </p><p>At the limit of an ideal undercomplete autoencoder, every possible code <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>z</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle z}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bf368e72c009decd9b6686ee84a375632e11de98" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.088ex; height:1.676ex;" alt="{\displaystyle z}" /></span> in the code space is used to encode a message <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span> that really appears in the distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{ref}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mu _{ref}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/15d15b590538ca3450d814c51e7707812a8c26b4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.046ex; height:2.343ex;" alt="{\displaystyle \mu _{ref}}" /></span>, and the decoder is also perfect: <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D_{\theta }(E_{\phi }(x))=x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo>=</mo> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D_{\theta }(E_{\phi }(x))=x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e539ece922d70a8af7087f56daa66c223f0f926f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:15.231ex; height:3.009ex;" alt="{\displaystyle D_{\theta }(E_{\phi }(x))=x}" /></span>. This ideal autoencoder can then be used to generate messages indistinguishable from real messages, by feeding its decoder arbitrary code <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>z</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle z}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bf368e72c009decd9b6686ee84a375632e11de98" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.088ex; height:1.676ex;" alt="{\displaystyle z}" /></span> and obtaining <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D_{\theta }(z)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>z</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D_{\theta }(z)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0f3eb8bf3238f6523cbceb2e714035f08aad6b84" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.825ex; height:2.843ex;" alt="{\displaystyle D_{\theta }(z)}" /></span>, which is a message that really appears in the distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{ref}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mu _{ref}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/15d15b590538ca3450d814c51e7707812a8c26b4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.046ex; height:2.343ex;" alt="{\displaystyle \mu _{ref}}" /></span>. </p><p>If the code space <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {Z}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {Z}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e9be1e2c3c82e7b8055fe26eb4cf2caac6f8ec73" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.783ex; height:2.176ex;" alt="{\displaystyle {\mathcal {Z}}}" /></span> has dimension larger than (<i>overcomplete</i>), or equal to, the message space <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8c7e5461c5286852df4ef652fca7e4b0b63030e9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.875ex; height:2.176ex;" alt="{\displaystyle {\mathcal {X}}}" /></span>, or the hidden units are given enough capacity, an autoencoder can learn the <a href="/wiki/Identity_function" title="Identity function">identity function</a> and become useless. However, experimental results found that overcomplete autoencoders might still <a href="/wiki/Feature_learning" title="Feature learning">learn useful features</a>.<sup id="cite_ref-bengio_11-0" class="reference"><a href="#cite_note-bengio-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> </p><p>In the ideal setting, the code dimension and the model capacity could be set on the basis of the complexity of the data distribution to be modeled. A standard way to do so is to add modifications to the basic autoencoder, to be detailed below.<sup id="cite_ref-:0_2-1" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Variations">Variations</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=5" title="Edit section: Variations"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Variational_autoencoder_(VAE)"><span id="Variational_autoencoder_.28VAE.29"></span>Variational autoencoder (VAE)</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=6" title="Edit section: Variational autoencoder (VAE)"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure typeof="mw:File/Thumb"><a href="/wiki/File:VAE_Basic.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/4/4a/VAE_Basic.png/300px-VAE_Basic.png" decoding="async" width="300" height="129" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/4/4a/VAE_Basic.png/450px-VAE_Basic.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/4/4a/VAE_Basic.png/600px-VAE_Basic.png 2x" data-file-width="1122" data-file-height="484" /></a><figcaption>The basic scheme of a variational autoencoder. The model receives <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span> as input. The encoder compresses it into the latent space. The decoder receives as input the information sampled from the latent space and produces <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {x'}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <msup> <mi>x</mi> <mo>′</mo> </msup> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {x'}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a223f94a9c179ffeb6e74e3208a27070a69f3f93" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.014ex; height:2.509ex;" alt="{\displaystyle {x'}}" /></span> as similar as possible to <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span>.</figcaption></figure> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951" /><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Variational_autoencoder" title="Variational autoencoder">Variational autoencoder</a></div> <p><a href="/wiki/Variational_autoencoder" title="Variational autoencoder">Variational autoencoders</a> (VAEs) belong to the families of <a href="/wiki/Variational_Bayesian_methods" title="Variational Bayesian methods">variational Bayesian methods</a>. Despite the architectural similarities with basic autoencoders, VAEs are architected with different goals and have a different mathematical formulation. The latent space is, in this case, composed of a mixture of distributions instead of fixed vectors. </p><p>Given an input dataset <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span> characterized by an unknown probability function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>P</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/89833156eff2c51bfb8750db3306a0544ce34e14" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.884ex; height:2.843ex;" alt="{\displaystyle P(x)}" /></span> and a multivariate latent encoding vector <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>z</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle z}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bf368e72c009decd9b6686ee84a375632e11de98" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.088ex; height:1.676ex;" alt="{\displaystyle z}" /></span>, the objective is to model the data as a distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{\theta }(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{\theta }(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1f0682b1f715edb12932100f59e93d012e4ec886" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:5.401ex; height:2.843ex;" alt="{\displaystyle p_{\theta }(x)}" /></span>, with <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>θ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \theta }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6e5ab2664b422d53eb0c7df3b87e1360d75ad9af" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.09ex; height:2.176ex;" alt="{\displaystyle \theta }" /></span> defined as the set of the network parameters so that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{\theta }(x)=\int _{z}p_{\theta }(x,z)dz}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>z</mi> </mrow> </msub> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <mi>d</mi> <mi>z</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{\theta }(x)=\int _{z}p_{\theta }(x,z)dz}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a18e1df9f777fee6a6ca3549ba5b1d014f689444" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; margin-left: -0.089ex; width:20.919ex; height:5.676ex;" alt="{\displaystyle p_{\theta }(x)=\int _{z}p_{\theta }(x,z)dz}" /></span>. </p> <div class="mw-heading mw-heading3"><h3 id="Sparse_autoencoder_(SAE)"><span id="Sparse_autoencoder_.28SAE.29"></span>Sparse autoencoder (SAE)</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=7" title="Edit section: Sparse autoencoder (SAE)"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div><p> Inspired by the <a href="/wiki/Sparse_coding" class="mw-redirect" title="Sparse coding">sparse coding</a> hypothesis in neuroscience, <i>sparse autoencoders</i> (SAE) are variants of autoencoders, such that the codes <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E_{\phi }(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle E_{\phi }(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d62337867b116b3895877e218f11ed8223e1d793" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:6.066ex; height:3.009ex;" alt="{\displaystyle E_{\phi }(x)}" /></span> for messages tend to be <i>sparse codes</i>, that is, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E_{\phi }(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle E_{\phi }(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d62337867b116b3895877e218f11ed8223e1d793" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:6.066ex; height:3.009ex;" alt="{\displaystyle E_{\phi }(x)}" /></span> is close to zero in most entries. Sparse autoencoders may include more (rather than fewer) hidden units than inputs, but only a small number of the hidden units are allowed to be active at the same time.<sup id="cite_ref-domingos_12-0" class="reference"><a href="#cite_note-domingos-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> Encouraging sparsity improves performance on classification tasks.<sup id="cite_ref-:1_13-0" class="reference"><a href="#cite_note-:1-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> </p><figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Autoencoder_sparso.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/8/83/Autoencoder_sparso.png/220px-Autoencoder_sparso.png" decoding="async" width="220" height="274" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/83/Autoencoder_sparso.png/330px-Autoencoder_sparso.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/83/Autoencoder_sparso.png/440px-Autoencoder_sparso.png 2x" data-file-width="442" data-file-height="550" /></a><figcaption>Simple schema of a single-layer sparse autoencoder. The hidden nodes in bright yellow are activated, while the light yellow ones are inactive. The activation depends on the input.</figcaption></figure> <p>There are two main ways to enforce sparsity. One way is to simply clamp all but the highest-k activations of the latent code to zero. This is the <b>k-sparse autoencoder</b>.<sup id="cite_ref-:1_13-1" class="reference"><a href="#cite_note-:1-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> </p><p>The k-sparse autoencoder inserts the following "k-sparse function" in the latent layer of a standard autoencoder:<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{k}(x_{1},...,x_{n})=(x_{1}b_{1},...,x_{n}b_{n})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{k}(x_{1},...,x_{n})=(x_{1}b_{1},...,x_{n}b_{n})}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e0230c8bdf0bbfd0c70e316b90d47ca6f180f5d6" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:33.416ex; height:2.843ex;" alt="{\displaystyle f_{k}(x_{1},...,x_{n})=(x_{1}b_{1},...,x_{n}b_{n})}" /></span>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{i}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle b_{i}=1}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e9be70edf0d554ce09ecff0b8c16e5a53a9c606a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.058ex; height:2.509ex;" alt="{\displaystyle b_{i}=1}" /></span> if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle |x_{i}|}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle |x_{i}|}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/92da0dacc0f9f069128bdd9b4c7de82cf81c2aed" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:3.423ex; height:2.843ex;" alt="{\displaystyle |x_{i}|}" /></span> ranks in the top k, and 0 otherwise. </p><p>Backpropagating through <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{k}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{k}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8a585492f646ca803bc408103a0c705dd67ab8b7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.228ex; height:2.509ex;" alt="{\displaystyle f_{k}}" /></span> is simple: set gradient to 0 for <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{i}=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle b_{i}=0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/5a1fdd6a52e2a95d37ce0fbccbb56dad6b4083a0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.058ex; height:2.509ex;" alt="{\displaystyle b_{i}=0}" /></span> entries, and keep gradient for <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{i}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>b</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle b_{i}=1}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e9be70edf0d554ce09ecff0b8c16e5a53a9c606a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.058ex; height:2.509ex;" alt="{\displaystyle b_{i}=1}" /></span> entries. This is essentially a generalized <a href="/wiki/Rectifier_(neural_networks)" title="Rectifier (neural networks)">ReLU</a> function.<sup id="cite_ref-:1_13-2" class="reference"><a href="#cite_note-:1-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> </p><p>The other way is a <a href="/wiki/Relaxation_(approximation)" title="Relaxation (approximation)">relaxed version</a> of the k-sparse autoencoder. Instead of forcing sparsity, we add a <b>sparsity regularization loss</b>, then optimize for<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )+\lambda L_{\text{sparse}}(\theta ,\phi )}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <munder> <mo movablelimits="true" form="prefix">min</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> </mrow> </munder> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mi>λ</mi> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>sparse</mtext> </mrow> </msub> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )+\lambda L_{\text{sparse}}(\theta ,\phi )}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e2b184d62f4d8db4863873eab8ed7a9e243f63a5" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:26.9ex; height:4.343ex;" alt="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )+\lambda L_{\text{sparse}}(\theta ,\phi )}" /></span>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda >0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>λ</mi> <mo>></mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \lambda >0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/eea25afc0351140f919cf791c49c1964b8b081de" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.616ex; height:2.176ex;" alt="{\displaystyle \lambda >0}" /></span> measures how much sparsity we want to enforce.<sup id="cite_ref-:6_14-0" class="reference"><a href="#cite_note-:6-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup> </p><p>Let the autoencoder architecture have <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>K</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle K}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2b76fce82a62ed5461908f0dc8f037de4e3686b0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.066ex; height:2.176ex;" alt="{\displaystyle K}" /></span> layers. To define a sparsity regularization loss, we need a "desired" sparsity <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {\rho }}_{k}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\hat {\rho }}_{k}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0f8abe27fcb896ed54b2f4926c7fe9a46dd25983" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.464ex; height:2.676ex;" alt="{\displaystyle {\hat {\rho }}_{k}}" /></span> for each layer, a weight <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{k}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{k}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ac808de3e1f01ed40c69d4a350035b19ab28445f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.753ex; height:2.009ex;" alt="{\displaystyle w_{k}}" /></span> for how much to enforce each sparsity, and a function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s:[0,1]\times [0,1]\to [0,\infty ]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>s</mi> <mo>:</mo> <mo stretchy="false">[</mo> <mn>0</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">]</mo> <mo>×</mo> <mo stretchy="false">[</mo> <mn>0</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">]</mo> <mo stretchy="false">→</mo> <mo stretchy="false">[</mo> <mn>0</mn> <mo>,</mo> <mi mathvariant="normal">∞</mi> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s:[0,1]\times [0,1]\to [0,\infty ]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0cd8c20f4b06b6eeddda1f90d5d1b5589ed8e6c3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:24.601ex; height:2.843ex;" alt="{\displaystyle s:[0,1]\times [0,1]\to [0,\infty ]}" /></span> to measure how much two sparsities differ. </p><p>For each input <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span>, let the actual sparsity of activation in each layer <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle k}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>k</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle k}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c3c9a2c7b599b37105512c5d570edc034056dd40" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.211ex; height:2.176ex;" alt="{\displaystyle k}" /></span> be<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \rho _{k}(x)={\frac {1}{n}}\sum _{i=1}^{n}a_{k,i}(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mi>n</mi> </mfrac> </mrow> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </munderover> <msub> <mi>a</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mo>,</mo> <mi>i</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \rho _{k}(x)={\frac {1}{n}}\sum _{i=1}^{n}a_{k,i}(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/484a083542fc48db444072c35bf7438b4f5f335f" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:21.37ex; height:6.843ex;" alt="{\displaystyle \rho _{k}(x)={\frac {1}{n}}\sum _{i=1}^{n}a_{k,i}(x)}" /></span>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle a_{k,i}(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>a</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mo>,</mo> <mi>i</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle a_{k,i}(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0d8db682dad885500aaebe6308322fbebe44d482" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:6.482ex; height:3.009ex;" alt="{\displaystyle a_{k,i}(x)}" /></span> is the activation in the <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>i</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle i}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/add78d8608ad86e54951b8c8bd6c8d8416533d20" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" /></span> -th neuron of the <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle k}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>k</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle k}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c3c9a2c7b599b37105512c5d570edc034056dd40" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.211ex; height:2.176ex;" alt="{\displaystyle k}" /></span> -th layer upon input <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span>. </p><p>The sparsity loss upon input <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87f9e315fd7e2ba406057a97300593c4802b53e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" /></span> for one layer is <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s({\hat {\rho }}_{k},\rho _{k}(x))}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>s</mi> <mo stretchy="false">(</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s({\hat {\rho }}_{k},\rho _{k}(x))}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3727a598f770abd4f65df3e0abe7ac4929ac9027" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.828ex; height:2.843ex;" alt="{\displaystyle s({\hat {\rho }}_{k},\rho _{k}(x))}" /></span>, and the sparsity regularization loss for the entire autoencoder is the expected weighted sum of sparsity losses:<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{sparse}}(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X}}\left[\sum _{k\in 1:K}w_{k}s({\hat {\rho }}_{k},\rho _{k}(x))\right]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>sparse</mtext> </mrow> </msub> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∼</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mrow> </msub> <mrow> <mo>[</mo> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mo>∈</mo> <mn>1</mn> <mo>:</mo> <mi>K</mi> </mrow> </munder> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mi>s</mi> <mo stretchy="false">(</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> </mrow> <mo>]</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{sparse}}(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X}}\left[\sum _{k\in 1:K}w_{k}s({\hat {\rho }}_{k},\rho _{k}(x))\right]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6384293aecb3ff69a6e8fee94b87546c533c4510" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:43.691ex; height:7.509ex;" alt="{\displaystyle L_{\text{sparse}}(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X}}\left[\sum _{k\in 1:K}w_{k}s({\hat {\rho }}_{k},\rho _{k}(x))\right]}" /></span>Typically, the function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>s</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/01d131dfd7673938b947072a13a9744fe997e632" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.09ex; height:1.676ex;" alt="{\displaystyle s}" /></span> is either the <a href="/wiki/Kullback%E2%80%93Leibler_divergence" title="Kullback–Leibler divergence">Kullback-Leibler (KL) divergence</a>, as<sup id="cite_ref-:1_13-3" class="reference"><a href="#cite_note-:1-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:6_14-1" class="reference"><a href="#cite_note-:6-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-15" class="reference"><a href="#cite_note-15"><span class="cite-bracket">[</span>15<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-16" class="reference"><a href="#cite_note-16"><span class="cite-bracket">[</span>16<span class="cite-bracket">]</span></a></sup> </p> <dl><dd><dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s(\rho ,{\hat {\rho }})=KL(\rho ||{\hat {\rho }})=\rho \log {\frac {\rho }{\hat {\rho }}}+(1-\rho )\log {\frac {1-\rho }{1-{\hat {\rho }}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>s</mi> <mo stretchy="false">(</mo> <mi>ρ</mi> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mo stretchy="false">)</mo> <mo>=</mo> <mi>K</mi> <mi>L</mi> <mo stretchy="false">(</mo> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mo stretchy="false">)</mo> <mo>=</mo> <mi>ρ</mi> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mfrac> </mrow> <mo>+</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <mi>ρ</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>1</mn> <mo>−</mo> <mi>ρ</mi> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s(\rho ,{\hat {\rho }})=KL(\rho ||{\hat {\rho }})=\rho \log {\frac {\rho }{\hat {\rho }}}+(1-\rho )\log {\frac {1-\rho }{1-{\hat {\rho }}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0460df1d442b4ccfb5d4e55243fe1ab5cb23e624" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:49.012ex; height:6.009ex;" alt="{\displaystyle s(\rho ,{\hat {\rho }})=KL(\rho ||{\hat {\rho }})=\rho \log {\frac {\rho }{\hat {\rho }}}+(1-\rho )\log {\frac {1-\rho }{1-{\hat {\rho }}}}}" /></span></dd></dl></dd></dl> <p>or the L1 loss, as <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s(\rho ,{\hat {\rho }})=|\rho -{\hat {\rho }}|}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>s</mi> <mo stretchy="false">(</mo> <mi>ρ</mi> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>ρ</mi> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s(\rho ,{\hat {\rho }})=|\rho -{\hat {\rho }}|}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/97093ba0c58ecc04bd3c26a793b254cbd5ca0136" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:16.321ex; height:2.843ex;" alt="{\displaystyle s(\rho ,{\hat {\rho }})=|\rho -{\hat {\rho }}|}" /></span>, or the L2 loss, as <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s(\rho ,{\hat {\rho }})=|\rho -{\hat {\rho }}|^{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>s</mi> <mo stretchy="false">(</mo> <mi>ρ</mi> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>ρ</mi> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>ρ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <msup> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle s(\rho ,{\hat {\rho }})=|\rho -{\hat {\rho }}|^{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ee874b463a23a182bbd5c4cee0e25766262effdb" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:17.376ex; height:3.343ex;" alt="{\displaystyle s(\rho ,{\hat {\rho }})=|\rho -{\hat {\rho }}|^{2}}" /></span>. </p><p>Alternatively, the sparsity regularization loss may be defined without reference to any "desired sparsity", but simply force as much sparsity as possible. In this case, one can define the sparsity regularization loss as <span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{sparse}}(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X}}\left[\sum _{k\in 1:K}w_{k}\|h_{k}\|\right]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>sparse</mtext> </mrow> </msub> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∼</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mrow> </msub> <mrow> <mo>[</mo> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mo>∈</mo> <mn>1</mn> <mo>:</mo> <mi>K</mi> </mrow> </munder> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> </mrow> <mo>]</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{sparse}}(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X}}\left[\sum _{k\in 1:K}w_{k}\|h_{k}\|\right]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8464dfd2872b84c2c4299a67fd9fcf1f57797d18" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:36.615ex; height:7.509ex;" alt="{\displaystyle L_{\text{sparse}}(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X}}\left[\sum _{k\in 1:K}w_{k}\|h_{k}\|\right]}" /></span>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{k}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>h</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle h_{k}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/78fe4f83c0bf136a170a0433c961330328b3596f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.428ex; height:2.509ex;" alt="{\displaystyle h_{k}}" /></span> is the activation vector in the <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle k}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>k</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle k}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c3c9a2c7b599b37105512c5d570edc034056dd40" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.211ex; height:2.176ex;" alt="{\displaystyle k}" /></span>-th layer of the autoencoder. The norm <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \|\cdot \|}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">‖</mo> <mo>⋅</mo> <mo fence="false" stretchy="false">‖</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \|\cdot \|}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/113f0d8fe6108fc1c5e9802f7c3f634f5480b3d1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.004ex; height:2.843ex;" alt="{\displaystyle \|\cdot \|}" /></span> is usually the L1 norm (giving the L1 sparse autoencoder) or the L2 norm (giving the L2 sparse autoencoder). </p> <div class="mw-heading mw-heading3"><h3 id="Denoising_autoencoder_(DAE)"><span id="Denoising_autoencoder_.28DAE.29"></span>Denoising autoencoder (DAE)</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=8" title="Edit section: Denoising autoencoder (DAE)"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Denoising-autoencoder.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/1/18/Denoising-autoencoder.png/250px-Denoising-autoencoder.png" decoding="async" width="220" height="106" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/1/18/Denoising-autoencoder.png/330px-Denoising-autoencoder.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/1/18/Denoising-autoencoder.png/500px-Denoising-autoencoder.png 2x" data-file-width="1743" data-file-height="840" /></a><figcaption>A schema of a denoising autoencoder</figcaption></figure> <p><i>Denoising autoencoders</i> (DAE) try to achieve a <i>good</i> representation by changing the <i>reconstruction criterion</i>.<sup id="cite_ref-:0_2-2" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:4_3-1" class="reference"><a href="#cite_note-:4-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> </p><p>A DAE, originally called a "robust autoassociative network" by Mark A. Kramer,<sup id="cite_ref-:13_17-0" class="reference"><a href="#cite_note-:13-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup> is trained by intentionally corrupting the inputs of a standard autoencoder during training. A noise process is defined by a probability distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{T}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>T</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mu _{T}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7c329840dcccf76c639e56c35ef3d56779d1fd5" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.791ex; height:2.176ex;" alt="{\displaystyle \mu _{T}}" /></span> over functions <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T:{\mathcal {X}}\to {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>T</mi> <mo>:</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo stretchy="false">→</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T:{\mathcal {X}}\to {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/16f818c3ae1d1c6ef3b735161eb885fea2e1a645" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:10.938ex; height:2.176ex;" alt="{\displaystyle T:{\mathcal {X}}\to {\mathcal {X}}}" /></span>. That is, the function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>T</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ec7200acd984a1d3a3d7dc455e262fbe54f7f6e0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.636ex; height:2.176ex;" alt="{\displaystyle T}" /></span> takes a message <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x\in {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x\in {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0b3cd596d560b9f3bc71615911c2438cbbf49c15" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.046ex; height:2.176ex;" alt="{\displaystyle x\in {\mathcal {X}}}" /></span>, and corrupts it to a noisy version <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T(x)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>T</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T(x)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1171c29b4c2b5575f50a4ea9313f90448a2cbe05" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.775ex; height:2.843ex;" alt="{\displaystyle T(x)}" /></span>. The function <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>T</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ec7200acd984a1d3a3d7dc455e262fbe54f7f6e0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.636ex; height:2.176ex;" alt="{\displaystyle T}" /></span> is selected randomly, with a probability distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{T}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>T</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mu _{T}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7c329840dcccf76c639e56c35ef3d56779d1fd5" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.791ex; height:2.176ex;" alt="{\displaystyle \mu _{T}}" /></span>. </p><p>Given a task <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (\mu _{\text{ref}},d)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>ref</mtext> </mrow> </msub> <mo>,</mo> <mi>d</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (\mu _{\text{ref}},d)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/eb846915e028b0a1c631d189a395264d84ac2158" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.571ex; height:2.843ex;" alt="{\displaystyle (\mu _{\text{ref}},d)}" /></span>, the problem of training a DAE is the optimization problem:<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X},T\sim \mu _{T}}[d(x,(D_{\theta }\circ E_{\phi }\circ T)(x))]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <munder> <mo movablelimits="true" form="prefix">min</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> </mrow> </munder> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∼</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo>,</mo> <mi>T</mi> <mo>∼</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>T</mi> </mrow> </msub> </mrow> </msub> <mo stretchy="false">[</mo> <mi>d</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mo stretchy="false">(</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> </mrow> </msub> <mo>∘</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo>∘</mo> <mi>T</mi> <mo stretchy="false">)</mo> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X},T\sim \mu _{T}}[d(x,(D_{\theta }\circ E_{\phi }\circ T)(x))]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2bdd1dd52947dc67ec42fba69511b4d28bdd7688" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:49.055ex; height:4.343ex;" alt="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )=\mathbb {\mathbb {E} } _{x\sim \mu _{X},T\sim \mu _{T}}[d(x,(D_{\theta }\circ E_{\phi }\circ T)(x))]}" /></span>That is, the optimal DAE should take any noisy message and attempt to recover the original message without noise, thus the name "denoising"<i>.</i> </p><p>Usually, the noise process <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>T</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ec7200acd984a1d3a3d7dc455e262fbe54f7f6e0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.636ex; height:2.176ex;" alt="{\displaystyle T}" /></span> is applied only during training and testing, not during downstream use. </p><p>The use of DAE depends on two assumptions: </p> <ul><li>There exist representations to the messages that are relatively stable and robust to the type of noise we are likely to encounter;</li> <li>The said representations capture structures in the input distribution that are useful for our purposes.<sup id="cite_ref-:4_3-2" class="reference"><a href="#cite_note-:4-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup></li></ul> <p>Example noise processes include: </p> <ul><li>additive isotropic <a href="/wiki/Additive_white_Gaussian_noise" title="Additive white Gaussian noise">Gaussian noise</a>,</li> <li>masking noise (a fraction of the input is randomly chosen and set to 0)</li> <li>salt-and-pepper noise (a fraction of the input is randomly chosen and randomly set to its minimum or maximum value).<sup id="cite_ref-:4_3-3" class="reference"><a href="#cite_note-:4-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup></li></ul> <div class="mw-heading mw-heading3"><h3 id="Contractive_autoencoder_(CAE)"><span id="Contractive_autoencoder_.28CAE.29"></span>Contractive autoencoder (CAE)</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=9" title="Edit section: Contractive autoencoder (CAE)"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A <i>contractive autoencoder</i> (CAE) adds the contractive regularization loss to the standard autoencoder loss:<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )+\lambda L_{\text{cont}}(\theta ,\phi )}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <munder> <mo movablelimits="true" form="prefix">min</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> </mrow> </munder> <mi>L</mi> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mi>λ</mi> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>cont</mtext> </mrow> </msub> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )+\lambda L_{\text{cont}}(\theta ,\phi )}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1d50899e5e474e145e04327418e6094e5661e28f" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:25.6ex; height:4.343ex;" alt="{\displaystyle \min _{\theta ,\phi }L(\theta ,\phi )+\lambda L_{\text{cont}}(\theta ,\phi )}" /></span>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda >0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>λ</mi> <mo>></mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \lambda >0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/eea25afc0351140f919cf791c49c1964b8b081de" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.616ex; height:2.176ex;" alt="{\displaystyle \lambda >0}" /></span> measures how much contractive-ness we want to enforce. The contractive regularization loss itself is defined as the expected square of <a href="/wiki/Frobenius_norm" class="mw-redirect" title="Frobenius norm">Frobenius norm</a> of the <a href="/wiki/Jacobian_matrix_and_determinant" title="Jacobian matrix and determinant">Jacobian matrix</a> of the encoder activations with respect to the input:<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{cont}}(\theta ,\phi )=\mathbb {E} _{x\sim \mu _{ref}}\|\nabla _{x}E_{\phi }(x)\|_{F}^{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>cont</mtext> </mrow> </msub> <mo stretchy="false">(</mo> <mi>θ</mi> <mo>,</mo> <mi>ϕ</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∼</mo> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>r</mi> <mi>e</mi> <mi>f</mi> </mrow> </msub> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> <msub> <mi mathvariant="normal">∇</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> </mrow> </msub> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msubsup> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>F</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{cont}}(\theta ,\phi )=\mathbb {E} _{x\sim \mu _{ref}}\|\nabla _{x}E_{\phi }(x)\|_{F}^{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b9ecf73ce3d00cc59e0dc569ae167c8c1e439269" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:33.416ex; height:3.509ex;" alt="{\displaystyle L_{\text{cont}}(\theta ,\phi )=\mathbb {E} _{x\sim \mu _{ref}}\|\nabla _{x}E_{\phi }(x)\|_{F}^{2}}" /></span>To understand what <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{cont}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>cont</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{cont}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/919e6a2bb3a36c6ba639b6e9edd360d17d1b2027" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.921ex; height:2.509ex;" alt="{\displaystyle L_{\text{cont}}}" /></span> measures, note the fact<span class="mwe-math-element"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \|E_{\phi }(x+\delta x)-E_{\phi }(x)\|_{2}\leq \|\nabla _{x}E_{\phi }(x)\|_{F}\|\delta x\|_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">‖</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <mi>δ</mi> <mi>x</mi> <mo stretchy="false">)</mo> <mo>−</mo> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>≤</mo> <mo fence="false" stretchy="false">‖</mo> <msub> <mi mathvariant="normal">∇</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> </mrow> </msub> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>F</mi> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> <mi>δ</mi> <mi>x</mi> <msub> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \|E_{\phi }(x+\delta x)-E_{\phi }(x)\|_{2}\leq \|\nabla _{x}E_{\phi }(x)\|_{F}\|\delta x\|_{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/20f744fa9cc9382d66d2f6a6bb05e3e31d9c9229" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:45.389ex; height:3.009ex;" alt="{\displaystyle \|E_{\phi }(x+\delta x)-E_{\phi }(x)\|_{2}\leq \|\nabla _{x}E_{\phi }(x)\|_{F}\|\delta x\|_{2}}" /></span>for any message <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x\in {\mathcal {X}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x\in {\mathcal {X}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0b3cd596d560b9f3bc71615911c2438cbbf49c15" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.046ex; height:2.176ex;" alt="{\displaystyle x\in {\mathcal {X}}}" /></span>, and small variation <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \delta x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>δ</mi> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \delta x}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d22318bef6d7358b79bd993321d65d7c1d3db9d4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.378ex; height:2.343ex;" alt="{\displaystyle \delta x}" /></span> in it. Thus, if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \|\nabla _{x}E_{\phi }(x)\|_{F}^{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">‖</mo> <msub> <mi mathvariant="normal">∇</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> </mrow> </msub> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>ϕ</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msubsup> <mo fence="false" stretchy="false">‖</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>F</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \|\nabla _{x}E_{\phi }(x)\|_{F}^{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b832eef5d8709f1a326dd9eba1609951ebc7ec19" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:12.963ex; height:3.176ex;" alt="{\displaystyle \|\nabla _{x}E_{\phi }(x)\|_{F}^{2}}" /></span> is small, it means that a small neighborhood of the message maps to a small neighborhood of its code. This is a desired property, as it means small variation in the message leads to small, perhaps even zero, variation in its code, like how two pictures may look the same even if they are not exactly the same. </p><p>The DAE can be understood as an infinitesimal limit of CAE: in the limit of small Gaussian input noise, DAEs make the reconstruction function resist small but finite-sized input perturbations, while CAEs make the extracted features resist infinitesimal input perturbations. </p> <div class="mw-heading mw-heading3"><h3 id="Minimum_description_length_autoencoder_(MDL-AE)"><span id="Minimum_description_length_autoencoder_.28MDL-AE.29"></span>Minimum description length autoencoder (MDL-AE)</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=10" title="Edit section: Minimum description length autoencoder (MDL-AE)"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A <i>minimum description length autoencoder</i> (MDL-AE) is an advanced variation of the traditional autoencoder, which leverages principles from information theory, specifically the <a href="/wiki/Minimum_description_length" title="Minimum description length">Minimum Description Length (MDL) principle</a>. The MDL principle posits that the best model for a dataset is the one that provides the shortest combined encoding of the model and the data. In the context of <a href="/wiki/Autoencoders" class="mw-redirect" title="Autoencoders">autoencoders</a>, this principle is applied to ensure that the learned representation is not only compact but also interpretable and efficient for reconstruction. </p><p>The MDL-AE seeks to minimize the total description length of the data, which includes the size of the <a href="/w/index.php?title=Latent_representation&action=edit&redlink=1" class="new" title="Latent representation (page does not exist)">latent representation</a> (code length) and the error in reconstructing the original data. The objective can be expressed as <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{code}}+L_{\text{error}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>code</mtext> </mrow> </msub> <mo>+</mo> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>error</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{code}}+L_{\text{error}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/04970ccd9de439b43708851e6d701aa20ae42d50" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:13.152ex; height:2.509ex;" alt="{\displaystyle L_{\text{code}}+L_{\text{error}}}" /></span>, where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{code}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>code</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{code}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/724dcb12347fd9f0bbe4582370dd161b2003d4a1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.011ex; height:2.509ex;" alt="{\displaystyle L_{\text{code}}}" /></span> represents the length of the compressed latent representation and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L_{\text{error}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>L</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>error</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle L_{\text{error}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/54fbecfb3bd1d0f30fa1c943dd068fdc662f32b2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.301ex; height:2.509ex;" alt="{\displaystyle L_{\text{error}}}" /></span> denotes the reconstruction error.<sup id="cite_ref-:5_18-0" class="reference"><a href="#cite_note-:5-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Concrete_autoencoder_(CAE)"><span id="Concrete_autoencoder_.28CAE.29"></span>Concrete autoencoder (CAE)</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=11" title="Edit section: Concrete autoencoder (CAE)"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The <i>concrete autoencoder</i> is designed for discrete feature selection.<sup id="cite_ref-19" class="reference"><a href="#cite_note-19"><span class="cite-bracket">[</span>19<span class="cite-bracket">]</span></a></sup> A concrete autoencoder forces the latent space to consist only of a user-specified number of features. The concrete autoencoder uses a continuous <a href="/wiki/Relaxation_(approximation)" title="Relaxation (approximation)">relaxation</a> of the <a href="/wiki/Categorical_distribution" title="Categorical distribution">categorical distribution</a> to allow gradients to pass through the feature selector layer, which makes it possible to use standard <a href="/wiki/Backpropagation" title="Backpropagation">backpropagation</a> to learn an optimal subset of input features that minimize reconstruction loss. </p> <div class="mw-heading mw-heading2"><h2 id="Advantages_of_depth">Advantages of depth</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=12" title="Edit section: Advantages of depth"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure typeof="mw:File/Thumb"><a href="/wiki/File:Autoencoder_structure.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/2/28/Autoencoder_structure.png/350px-Autoencoder_structure.png" decoding="async" width="350" height="262" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/2/28/Autoencoder_structure.png/525px-Autoencoder_structure.png 1.5x, //upload.wikimedia.org/wikipedia/commons/2/28/Autoencoder_structure.png 2x" data-file-width="677" data-file-height="506" /></a><figcaption>Schematic structure of an autoencoder with 3 fully connected hidden layers. The code (z, or h for reference in the text) is the most internal layer.</figcaption></figure> <p>Autoencoders are often trained with a single-layer encoder and a single-layer decoder, but using many-layered (deep) encoders and decoders offers many advantages.<sup id="cite_ref-:0_2-3" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p> <ul><li>Depth can exponentially reduce the computational cost of representing some functions.</li> <li>Depth can exponentially decrease the amount of training data needed to learn some functions.</li> <li>Experimentally, deep autoencoders yield better compression compared to shallow or linear autoencoders.<sup id="cite_ref-:7_10-1" class="reference"><a href="#cite_note-:7-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup></li></ul> <div class="mw-heading mw-heading3"><h3 id="Training">Training</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=13" title="Edit section: Training"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p><a href="/wiki/Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a> developed the <a href="/wiki/Deep_belief_network" title="Deep belief network">deep belief network</a> technique for training many-layered deep autoencoders. His method involves treating each neighboring set of two layers as a <a href="/wiki/Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">restricted Boltzmann machine</a> so that pretraining approximates a good solution, then using backpropagation to fine-tune the results.<sup id="cite_ref-:7_10-2" class="reference"><a href="#cite_note-:7-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> </p><p>Researchers have debated whether joint training (i.e. training the whole architecture together with a single global reconstruction objective to optimize) would be better for deep auto-encoders.<sup id="cite_ref-:9_20-0" class="reference"><a href="#cite_note-:9-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup> A 2015 study showed that joint training learns better data models along with more representative features for classification as compared to the layerwise method.<sup id="cite_ref-:9_20-1" class="reference"><a href="#cite_note-:9-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup> However, their experiments showed that the success of joint training depends heavily on the regularization strategies adopted.<sup id="cite_ref-:9_20-2" class="reference"><a href="#cite_note-:9-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-21" class="reference"><a href="#cite_note-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="History">History</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=14" title="Edit section: History"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>(Oja, 1982)<sup id="cite_ref-22" class="reference"><a href="#cite_note-22"><span class="cite-bracket">[</span>22<span class="cite-bracket">]</span></a></sup> noted that PCA is equivalent to a neural network with one hidden layer with identity activation function. In the language of autoencoding, the input-to-hidden module is the encoder, and the hidden-to-output module is the decoder. Subsequently, in (Baldi and Hornik, 1989)<sup id="cite_ref-auto_23-0" class="reference"><a href="#cite_note-auto-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup> and (Kramer, 1991)<sup id="cite_ref-:12_9-1" class="reference"><a href="#cite_note-:12-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> generalized PCA to autoencoders, which they termed as "nonlinear PCA". </p><p>Immediately after the resurgence of neural networks in the 1980s, it was suggested in 1986<sup id="cite_ref-24" class="reference"><a href="#cite_note-24"><span class="cite-bracket">[</span>24<span class="cite-bracket">]</span></a></sup> that a neural network be put in "auto-association mode". This was then implemented in (Harrison, 1987)<sup id="cite_ref-25" class="reference"><a href="#cite_note-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup> and (Elman, Zipser, 1988)<sup id="cite_ref-26" class="reference"><a href="#cite_note-26"><span class="cite-bracket">[</span>26<span class="cite-bracket">]</span></a></sup> for speech and in (Cottrell, Munro, Zipser, 1987)<sup id="cite_ref-27" class="reference"><a href="#cite_note-27"><span class="cite-bracket">[</span>27<span class="cite-bracket">]</span></a></sup> for images.<sup id="cite_ref-:14_28-0" class="reference"><a href="#cite_note-:14-28"><span class="cite-bracket">[</span>28<span class="cite-bracket">]</span></a></sup> In (Hinton, Salakhutdinov, 2006),<sup id="cite_ref-:72_29-0" class="reference"><a href="#cite_note-:72-29"><span class="cite-bracket">[</span>29<span class="cite-bracket">]</span></a></sup> <a href="/wiki/Deep_belief_network" title="Deep belief network">deep belief networks</a> were developed. These train a pair <a href="/wiki/Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">restricted Boltzmann machines</a> as encoder-decoder pairs, then train another pair on the latent representation of the first pair, and so on.<sup id="cite_ref-scholar_30-0" class="reference"><a href="#cite_note-scholar-30"><span class="cite-bracket">[</span>30<span class="cite-bracket">]</span></a></sup> </p><p>The first applications of AE date to early 1990s.<sup id="cite_ref-:0_2-4" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-31" class="reference"><a href="#cite_note-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:5_18-1" class="reference"><a href="#cite_note-:5-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup> Their most traditional application was <a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">dimensionality reduction</a> or <a href="/wiki/Feature_learning" title="Feature learning">feature learning</a>, but the concept became widely used for learning <a href="/wiki/Generative_model" title="Generative model">generative models</a> of data.<sup id="cite_ref-VAE_32-0" class="reference"><a href="#cite_note-VAE-32"><span class="cite-bracket">[</span>32<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-gan_faces_33-0" class="reference"><a href="#cite_note-gan_faces-33"><span class="cite-bracket">[</span>33<span class="cite-bracket">]</span></a></sup> Some of the most powerful <a href="/wiki/Artificial_intelligence" title="Artificial intelligence">AIs</a> in the 2010s involved autoencoder modules as a component of larger AI systems, such as VAE in <a href="/wiki/Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a>, discrete VAE in Transformer-based image generators like <a href="/wiki/DALL-E" title="DALL-E">DALL-E 1</a>, etc. </p><p>During the early days, when the terminology was uncertain, the autoencoder has also been called identity mapping,<sup id="cite_ref-auto_23-1" class="reference"><a href="#cite_note-auto-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:12_9-2" class="reference"><a href="#cite_note-:12-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> auto-associating,<sup id="cite_ref-34" class="reference"><a href="#cite_note-34"><span class="cite-bracket">[</span>34<span class="cite-bracket">]</span></a></sup> <a href="/wiki/Self-supervised_learning" title="Self-supervised learning">self-supervised</a> <a href="/wiki/Backpropagation" title="Backpropagation">backpropagation</a>,<sup id="cite_ref-:12_9-3" class="reference"><a href="#cite_note-:12-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> or Diabolo network.<sup id="cite_ref-35" class="reference"><a href="#cite_note-35"><span class="cite-bracket">[</span>35<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-bengio_11-1" class="reference"><a href="#cite_note-bengio-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Applications">Applications</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=15" title="Edit section: Applications"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The two main applications of autoencoders are <a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">dimensionality reduction</a> and <a href="/wiki/Information_retrieval" title="Information retrieval">information retrieval</a> (or <a href="/wiki/Content-addressable_memory" title="Content-addressable memory">associative memory</a>),<sup id="cite_ref-:0_2-5" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> but modern variations have been applied to other tasks. </p> <div class="mw-heading mw-heading3"><h3 id="Dimensionality_reduction">Dimensionality reduction</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=16" title="Edit section: Dimensionality reduction"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:PCA_vs_Linear_Autoencoder.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/0/0b/PCA_vs_Linear_Autoencoder.png/220px-PCA_vs_Linear_Autoencoder.png" decoding="async" width="220" height="110" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/0/0b/PCA_vs_Linear_Autoencoder.png/330px-PCA_vs_Linear_Autoencoder.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/0/0b/PCA_vs_Linear_Autoencoder.png/440px-PCA_vs_Linear_Autoencoder.png 2x" data-file-width="576" data-file-height="288" /></a><figcaption>Plot of the first two Principal Components (left) and a two-dimension hidden layer of a Linear Autoencoder (Right) applied to the <a href="/wiki/Fashion_MNIST" title="Fashion MNIST">Fashion MNIST</a> dataset.<sup id="cite_ref-:10_36-0" class="reference"><a href="#cite_note-:10-36"><span class="cite-bracket">[</span>36<span class="cite-bracket">]</span></a></sup> The two models being both linear learn to span the same subspace. The projection of the data points is indeed identical, apart from rotation of the subspace. While PCA selects a specific orientation up to reflections in the general case, the cost function of a simple autoencoder is invariant to rotations of the latent space.</figcaption></figure><p><a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a> was one of the first <a href="/wiki/Deep_learning" title="Deep learning">deep learning</a> applications.<sup id="cite_ref-:0_2-6" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p><p>For Hinton's 2006 study,<sup id="cite_ref-:7_10-3" class="reference"><a href="#cite_note-:7-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> he pretrained a multi-layer autoencoder with a stack of <a href="/wiki/Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">RBMs</a> and then used their weights to initialize a deep autoencoder with gradually smaller hidden layers until hitting a bottleneck of 30 neurons. The resulting 30 dimensions of the code yielded a smaller reconstruction error compared to the first 30 components of a principal component analysis (PCA), and learned a representation that was qualitatively easier to interpret, clearly separating data clusters.<sup id="cite_ref-:0_2-7" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:7_10-4" class="reference"><a href="#cite_note-:7-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> </p><p>Representing dimensions can improve performance on tasks such as classification.<sup id="cite_ref-:0_2-8" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> Indeed, the hallmark of dimensionality reduction is to place semantically related examples near each other.<sup id="cite_ref-:3_37-0" class="reference"><a href="#cite_note-:3-37"><span class="cite-bracket">[</span>37<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading4"><h4 id="Principal_component_analysis">Principal component analysis</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=17" title="Edit section: Principal component analysis"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Reconstruction_autoencoders_vs_PCA.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Reconstruction_autoencoders_vs_PCA.png/220px-Reconstruction_autoencoders_vs_PCA.png" decoding="async" width="220" height="48" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Reconstruction_autoencoders_vs_PCA.png/330px-Reconstruction_autoencoders_vs_PCA.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Reconstruction_autoencoders_vs_PCA.png/440px-Reconstruction_autoencoders_vs_PCA.png 2x" data-file-width="2008" data-file-height="441" /></a><figcaption>Reconstruction of 28x28pixel images by an Autoencoder with a code size of two (two-units hidden layer) and the reconstruction from the first two Principal Components of PCA. Images come from the <a href="/w/index.php?title=Fashion_MNIST_dataset&action=edit&redlink=1" class="new" title="Fashion MNIST dataset (page does not exist)">Fashion MNIST dataset</a>.<sup id="cite_ref-:10_36-1" class="reference"><a href="#cite_note-:10-36"><span class="cite-bracket">[</span>36<span class="cite-bracket">]</span></a></sup></figcaption></figure> <p>If linear activations are used, or only a single sigmoid hidden layer, then the optimal solution to an autoencoder is strongly related to <a href="/wiki/Principal_component_analysis" title="Principal component analysis">principal component analysis</a> (PCA).<sup id="cite_ref-:14_28-1" class="reference"><a href="#cite_note-:14-28"><span class="cite-bracket">[</span>28<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-38" class="reference"><a href="#cite_note-38"><span class="cite-bracket">[</span>38<span class="cite-bracket">]</span></a></sup> The weights of an autoencoder with a single hidden layer of size <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/81eac1e205430d1f40810df36a0edffdc367af36" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:1.259ex; height:2.009ex;" alt="{\displaystyle p}" /></span> (where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/81eac1e205430d1f40810df36a0edffdc367af36" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:1.259ex; height:2.009ex;" alt="{\displaystyle p}" /></span> is less than the size of the input) span the same vector subspace as the one spanned by the first <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/81eac1e205430d1f40810df36a0edffdc367af36" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:1.259ex; height:2.009ex;" alt="{\displaystyle p}" /></span> principal components, and the output of the autoencoder is an orthogonal projection onto this subspace. The autoencoder weights are not equal to the principal components, and are generally not orthogonal, yet the principal components may be recovered from them using the <a href="/wiki/Singular_value_decomposition" title="Singular value decomposition">singular value decomposition</a>.<sup id="cite_ref-39" class="reference"><a href="#cite_note-39"><span class="cite-bracket">[</span>39<span class="cite-bracket">]</span></a></sup> </p><p>However, the potential of autoencoders resides in their non-linearity, allowing the model to learn more powerful generalizations compared to PCA, and to reconstruct the input with significantly lower information loss.<sup id="cite_ref-:7_10-5" class="reference"><a href="#cite_note-:7-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Information_retrieval_and_Search_engine_optimization">Information retrieval and Search engine optimization</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=18" title="Edit section: Information retrieval and Search engine optimization"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p><a href="/wiki/Information_retrieval" title="Information retrieval">Information retrieval</a> benefits particularly from <a href="/wiki/Dimensionality_reduction" title="Dimensionality reduction">dimensionality reduction</a> in that search can become more efficient in certain kinds of low dimensional spaces. Autoencoders were indeed applied to semantic hashing, proposed by <a href="/wiki/Russ_Salakhutdinov" class="mw-redirect" title="Russ Salakhutdinov">Salakhutdinov</a> and Hinton in 2007.<sup id="cite_ref-:3_37-1" class="reference"><a href="#cite_note-:3-37"><span class="cite-bracket">[</span>37<span class="cite-bracket">]</span></a></sup> By training the algorithm to produce a low-dimensional binary code, all database entries could be stored in a <a href="/wiki/Hash_table" title="Hash table">hash table</a> mapping binary code vectors to entries. This table would then support information retrieval by returning all entries with the same binary code as the query, or slightly less similar entries by flipping some bits from the query encoding. </p><p>The encoder-decoder architecture, often used in natural language processing and neural networks, can be scientifically applied in the field of SEO (Search Engine Optimization) in various ways: </p> <ol><li><b>Text Processing</b>: By using an autoencoder, it's possible to compress the text of web pages into a more compact vector representation. This can help reduce page loading times and improve indexing by search engines.</li> <li class="mw-empty-elt"></li> <li><b>Noise Reduction</b>: Autoencoders can be used to remove noise from the textual data of web pages. This can lead to a better understanding of the content by search engines, thereby enhancing ranking in search engine result pages.</li> <li class="mw-empty-elt"></li> <li><b>Meta Tag and Snippet Generation</b>: Autoencoders can be trained to automatically generate meta tags, snippets, and descriptions for web pages using the page content. This can optimize the presentation in search results, increasing the Click-Through Rate (CTR).</li> <li class="mw-empty-elt"></li> <li><b>Content Clustering</b>: Using an autoencoder, web pages with similar content can be automatically grouped together. This can help organize the website logically and improve navigation, potentially positively affecting user experience and search engine rankings.</li> <li class="mw-empty-elt"></li> <li><b>Generation of Related Content</b>: An autoencoder can be employed to generate content related to what is already present on the site. This can enhance the website's attractiveness to search engines and provide users with additional relevant information.</li> <li class="mw-empty-elt"></li> <li><b>Keyword Detection</b>: Autoencoders can be trained to identify keywords and important concepts within the content of web pages. This can assist in optimizing keyword usage for better indexing.</li> <li class="mw-empty-elt"></li> <li><b>Semantic Search</b>: By using autoencoder techniques, semantic representation models of content can be created. These models can be used to enhance search engines' understanding of the themes covered in web pages.</li></ol> <p>In essence, the encoder-decoder architecture or autoencoders can be leveraged in SEO to optimize web page content, improve their indexing, and enhance their appeal to both search engines and users. </p> <div class="mw-heading mw-heading3"><h3 id="Anomaly_detection">Anomaly detection</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=19" title="Edit section: Anomaly detection"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Another application for autoencoders is <a href="/wiki/Anomaly_detection" title="Anomaly detection">anomaly detection</a>.<sup id="cite_ref-:13_17-1" class="reference"><a href="#cite_note-:13-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-40" class="reference"><a href="#cite_note-40"><span class="cite-bracket">[</span>40<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-41" class="reference"><a href="#cite_note-41"><span class="cite-bracket">[</span>41<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:8_42-0" class="reference"><a href="#cite_note-:8-42"><span class="cite-bracket">[</span>42<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-43" class="reference"><a href="#cite_note-43"><span class="cite-bracket">[</span>43<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-44" class="reference"><a href="#cite_note-44"><span class="cite-bracket">[</span>44<span class="cite-bracket">]</span></a></sup> By learning to replicate the most salient features in the training data under some of the constraints described previously, the model is encouraged to learn to precisely reproduce the most frequently observed characteristics. When facing anomalies, the model should worsen its reconstruction performance. In most cases, only data with normal instances are used to train the autoencoder; in others, the frequency of anomalies is small compared to the observation set so that its contribution to the learned representation could be ignored. After training, the autoencoder will accurately reconstruct "normal" data, while failing to do so with unfamiliar anomalous data.<sup id="cite_ref-:8_42-1" class="reference"><a href="#cite_note-:8-42"><span class="cite-bracket">[</span>42<span class="cite-bracket">]</span></a></sup> Reconstruction error (the error between the original data and its low dimensional reconstruction) is used as an anomaly score to detect anomalies.<sup id="cite_ref-:8_42-2" class="reference"><a href="#cite_note-:8-42"><span class="cite-bracket">[</span>42<span class="cite-bracket">]</span></a></sup> </p><p>Recent literature has however shown that certain autoencoding models can, counterintuitively, be very good at reconstructing anomalous examples and consequently not able to reliably perform anomaly detection.<sup id="cite_ref-45" class="reference"><a href="#cite_note-45"><span class="cite-bracket">[</span>45<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-46" class="reference"><a href="#cite_note-46"><span class="cite-bracket">[</span>46<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Image_processing">Image processing</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=20" title="Edit section: Image processing"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The characteristics of autoencoders are useful in image processing. </p><p>One example can be found in lossy <a href="/wiki/Image_compression" title="Image compression">image compression</a>, where autoencoders outperformed other approaches and proved competitive against <a href="/wiki/JPEG_2000" title="JPEG 2000">JPEG 2000</a>.<sup id="cite_ref-47" class="reference"><a href="#cite_note-47"><span class="cite-bracket">[</span>47<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-48" class="reference"><a href="#cite_note-48"><span class="cite-bracket">[</span>48<span class="cite-bracket">]</span></a></sup> </p><p>Another useful application of autoencoders in image preprocessing is <a href="/wiki/Image_denoising" class="mw-redirect" title="Image denoising">image denoising</a>.<sup id="cite_ref-49" class="reference"><a href="#cite_note-49"><span class="cite-bracket">[</span>49<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-50" class="reference"><a href="#cite_note-50"><span class="cite-bracket">[</span>50<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-51" class="reference"><a href="#cite_note-51"><span class="cite-bracket">[</span>51<span class="cite-bracket">]</span></a></sup> </p><p>Autoencoders found use in more demanding contexts such as <a href="/wiki/Medical_imaging" title="Medical imaging">medical imaging</a> where they have been used for <a href="/wiki/Image_denoising" class="mw-redirect" title="Image denoising">image denoising</a><sup id="cite_ref-52" class="reference"><a href="#cite_note-52"><span class="cite-bracket">[</span>52<span class="cite-bracket">]</span></a></sup> as well as <a href="/wiki/Super-resolution" class="mw-redirect" title="Super-resolution">super-resolution</a>.<sup id="cite_ref-53" class="reference"><a href="#cite_note-53"><span class="cite-bracket">[</span>53<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-54" class="reference"><a href="#cite_note-54"><span class="cite-bracket">[</span>54<span class="cite-bracket">]</span></a></sup> In image-assisted diagnosis, experiments have applied autoencoders for <a href="/wiki/Breast_cancer" title="Breast cancer">breast cancer</a> detection<sup id="cite_ref-55" class="reference"><a href="#cite_note-55"><span class="cite-bracket">[</span>55<span class="cite-bracket">]</span></a></sup> and for modelling the relation between the cognitive decline of <a href="/wiki/Alzheimer%27s_disease" title="Alzheimer's disease">Alzheimer's disease</a> and the latent features of an autoencoder trained with <a href="/wiki/MRI" class="mw-redirect" title="MRI">MRI</a>.<sup id="cite_ref-56" class="reference"><a href="#cite_note-56"><span class="cite-bracket">[</span>56<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Drug_discovery">Drug discovery</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=21" title="Edit section: Drug discovery"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>In 2019 molecules generated with variational autoencoders were validated experimentally in mice.<sup id="cite_ref-57" class="reference"><a href="#cite_note-57"><span class="cite-bracket">[</span>57<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-58" class="reference"><a href="#cite_note-58"><span class="cite-bracket">[</span>58<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Popularity_prediction">Popularity prediction</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=22" title="Edit section: Popularity prediction"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Recently, a stacked autoencoder framework produced promising results in predicting popularity of social media posts,<sup id="cite_ref-59" class="reference"><a href="#cite_note-59"><span class="cite-bracket">[</span>59<span class="cite-bracket">]</span></a></sup> which is helpful for online advertising strategies. </p> <div class="mw-heading mw-heading3"><h3 id="Machine_translation">Machine translation</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=23" title="Edit section: Machine translation"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Autoencoders have been applied to <a href="/wiki/Machine_translation" title="Machine translation">machine translation</a>, which is usually referred to as <a href="/wiki/Neural_machine_translation" title="Neural machine translation">neural machine translation</a> (NMT).<sup id="cite_ref-60" class="reference"><a href="#cite_note-60"><span class="cite-bracket">[</span>60<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-61" class="reference"><a href="#cite_note-61"><span class="cite-bracket">[</span>61<span class="cite-bracket">]</span></a></sup> Unlike traditional autoencoders, the output does not match the input - it is in another language. In NMT, texts are treated as sequences to be encoded into the learning procedure, while on the decoder side sequences in the target language(s) are generated. <a href="/wiki/Language" title="Language">Language</a>-specific autoencoders incorporate further <a href="/wiki/Linguistic" class="mw-redirect" title="Linguistic">linguistic</a> features into the learning procedure, such as Chinese decomposition features.<sup id="cite_ref-62" class="reference"><a href="#cite_note-62"><span class="cite-bracket">[</span>62<span class="cite-bracket">]</span></a></sup> Machine translation is rarely still done with autoencoders, due to the availability of more effective <a href="/wiki/Transformer_(machine_learning_model)" class="mw-redirect" title="Transformer (machine learning model)">transformer</a> networks. </p> <div class="mw-heading mw-heading3"><h3 id="Communication_Systems">Communication Systems</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=24" title="Edit section: Communication Systems"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Autoencoders in communication systems are important because they help in encoding data into a more resilient representation for channel impairments, which is crucial for transmitting information while minimizing errors. In Addition, AE-based systems can optimize end-to-end communication performance. This approach can solve the several limitations of designing communication systems such as the inherent difficulty in accurately modeling the complex behavior of real-world channels.<sup id="cite_ref-63" class="reference"><a href="#cite_note-63"><span class="cite-bracket">[</span>63<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=25" title="Edit section: See also"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Representation_learning" class="mw-redirect" title="Representation learning">Representation learning</a></li> <li><a href="/wiki/Sparse_dictionary_learning" title="Sparse dictionary learning">Sparse dictionary learning</a></li> <li><a href="/wiki/Deep_learning" title="Deep learning">Deep learning</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=26" title="Edit section: Further reading"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite id="CITEREFBankKoenigsteinGiryes2023" class="citation book cs1">Bank, Dor; Koenigstein, Noam; Giryes, Raja (2023). "Autoencoders". <i>Machine Learning for Data Science Handbook</i>. Cham: Springer International Publishing. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-031-24628-9_16">10.1007/978-3-031-24628-9_16</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-031-24627-2" title="Special:BookSources/978-3-031-24627-2"><bdi>978-3-031-24627-2</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Autoencoders&rft.btitle=Machine+Learning+for+Data+Science+Handbook&rft.place=Cham&rft.pub=Springer+International+Publishing&rft.date=2023&rft_id=info%3Adoi%2F10.1007%2F978-3-031-24628-9_16&rft.isbn=978-3-031-24627-2&rft.aulast=Bank&rft.aufirst=Dor&rft.au=Koenigstein%2C+Noam&rft.au=Giryes%2C+Raja&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGoodfellowBengioCourville2016" class="citation book cs1">Goodfellow, Ian; Bengio, Yoshua; Courville, Aaron (2016). <a rel="nofollow" class="external text" href="https://www.deeplearningbook.org/contents/autoencoders.html">"14. Autoencoders"</a>. <i>Deep learning</i>. Adaptive computation and machine learning. Cambridge, Mass: The MIT press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-262-03561-3" title="Special:BookSources/978-0-262-03561-3"><bdi>978-0-262-03561-3</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=14.+Autoencoders&rft.btitle=Deep+learning&rft.place=Cambridge%2C+Mass&rft.series=Adaptive+computation+and+machine+learning&rft.pub=The+MIT+press&rft.date=2016&rft.isbn=978-0-262-03561-3&rft.aulast=Goodfellow&rft.aufirst=Ian&rft.au=Bengio%2C+Yoshua&rft.au=Courville%2C+Aaron&rft_id=https%3A%2F%2Fwww.deeplearningbook.org%2Fcontents%2Fautoencoders.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></li></ul> <div class="mw-heading mw-heading2"><h2 id="References">References</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Autoencoder&action=edit&section=27" title="Edit section: References"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em;"> <ol class="references"> <li id="cite_note-1"><span class="mw-cite-backlink"><b><a href="#cite_ref-1">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBankKoenigsteinGiryes2023" class="citation book cs1">Bank, Dor; Koenigstein, Noam; Giryes, Raja (2023). <a rel="nofollow" class="external text" href="https://link.springer.com/chapter/10.1007/978-3-031-24628-9_16">"Autoencoders"</a>. In Rokach, Lior; Maimon, Oded; Shmueli, Erez (eds.). <i>Machine learning for data science handbook</i>. pp. <span class="nowrap">353–</span>374. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-031-24628-9_16">10.1007/978-3-031-24628-9_16</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-031-24627-2" title="Special:BookSources/978-3-031-24627-2"><bdi>978-3-031-24627-2</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Autoencoders&rft.btitle=Machine+learning+for+data+science+handbook&rft.pages=%3Cspan+class%3D%22nowrap%22%3E353-%3C%2Fspan%3E374&rft.date=2023&rft_id=info%3Adoi%2F10.1007%2F978-3-031-24628-9_16&rft.isbn=978-3-031-24627-2&rft.aulast=Bank&rft.aufirst=Dor&rft.au=Koenigstein%2C+Noam&rft.au=Giryes%2C+Raja&rft_id=https%3A%2F%2Flink.springer.com%2Fchapter%2F10.1007%2F978-3-031-24628-9_16&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:0-2"><span class="mw-cite-backlink">^ <a href="#cite_ref-:0_2-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:0_2-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:0_2-2"><sup><i><b>c</b></i></sup></a> <a href="#cite_ref-:0_2-3"><sup><i><b>d</b></i></sup></a> <a href="#cite_ref-:0_2-4"><sup><i><b>e</b></i></sup></a> <a href="#cite_ref-:0_2-5"><sup><i><b>f</b></i></sup></a> <a href="#cite_ref-:0_2-6"><sup><i><b>g</b></i></sup></a> <a href="#cite_ref-:0_2-7"><sup><i><b>h</b></i></sup></a> <a href="#cite_ref-:0_2-8"><sup><i><b>i</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGoodfellowBengioCourville2016" class="citation book cs1">Goodfellow, Ian; Bengio, Yoshua; Courville, Aaron (2016). <a rel="nofollow" class="external text" href="http://www.deeplearningbook.org"><i>Deep Learning</i></a>. MIT Press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0262035613" title="Special:BookSources/978-0262035613"><bdi>978-0262035613</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Deep+Learning&rft.pub=MIT+Press&rft.date=2016&rft.isbn=978-0262035613&rft.aulast=Goodfellow&rft.aufirst=Ian&rft.au=Bengio%2C+Yoshua&rft.au=Courville%2C+Aaron&rft_id=http%3A%2F%2Fwww.deeplearningbook.org&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:4-3"><span class="mw-cite-backlink">^ <a href="#cite_ref-:4_3-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:4_3-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:4_3-2"><sup><i><b>c</b></i></sup></a> <a href="#cite_ref-:4_3-3"><sup><i><b>d</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFVincentLarochelle2010" class="citation journal cs1">Vincent, Pascal; Larochelle, Hugo (2010). "Stacked Denoising Autoencoders: Learning Useful Representations in a Deep Network with a Local Denoising Criterion". <i>Journal of Machine Learning Research</i>. <b>11</b>: <span class="nowrap">3371–</span>3408.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Machine+Learning+Research&rft.atitle=Stacked+Denoising+Autoencoders%3A+Learning+Useful+Representations+in+a+Deep+Network+with+a+Local+Denoising+Criterion&rft.volume=11&rft.pages=%3Cspan+class%3D%22nowrap%22%3E3371-%3C%2Fspan%3E3408&rft.date=2010&rft.aulast=Vincent&rft.aufirst=Pascal&rft.au=Larochelle%2C+Hugo&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:11-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-:11_4-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFWellingKingma2019" class="citation journal cs1">Welling, Max; Kingma, Diederik P. (2019). "An Introduction to Variational Autoencoders". <i>Foundations and Trends in Machine Learning</i>. <b>12</b> (4): <span class="nowrap">307–</span>392. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1906.02691">1906.02691</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2019arXiv190602691K">2019arXiv190602691K</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1561%2F2200000056">10.1561/2200000056</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:174802445">174802445</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Foundations+and+Trends+in+Machine+Learning&rft.atitle=An+Introduction+to+Variational+Autoencoders&rft.volume=12&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E307-%3C%2Fspan%3E392&rft.date=2019&rft_id=info%3Aarxiv%2F1906.02691&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A174802445%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1561%2F2200000056&rft_id=info%3Abibcode%2F2019arXiv190602691K&rft.aulast=Welling&rft.aufirst=Max&rft.au=Kingma%2C+Diederik+P.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-5">^</a></b></span> <span class="reference-text">Hinton GE, Krizhevsky A, Wang SD. <a rel="nofollow" class="external text" href="http://www.cs.toronto.edu/~fritz/absps/transauto6.pdf">Transforming auto-encoders.</a> In International Conference on Artificial Neural Networks 2011 Jun 14 (pp. 44-51). Springer, Berlin, Heidelberg.</span> </li> <li id="cite_note-:2-6"><span class="mw-cite-backlink">^ <a href="#cite_ref-:2_6-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:2_6-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGéron2019" class="citation book cs1">Géron, Aurélien (2019). <i>Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow</i>. Canada: O’Reilly Media, Inc. pp. <span class="nowrap">739–</span>740.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Hands-On+Machine+Learning+with+Scikit-Learn%2C+Keras%2C+and+TensorFlow&rft.place=Canada&rft.pages=%3Cspan+class%3D%22nowrap%22%3E739-%3C%2Fspan%3E740&rft.pub=O%E2%80%99Reilly+Media%2C+Inc.&rft.date=2019&rft.aulast=G%C3%A9ron&rft.aufirst=Aur%C3%A9lien&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFLiouHuangYang2008" class="citation journal cs1">Liou, Cheng-Yuan; Huang, Jau-Chi; Yang, Wen-Chie (2008). <a rel="nofollow" class="external text" href="http://ntur.lib.ntu.edu.tw//handle/246246/155195">"Modeling word perception using the Elman network"</a>. <i>Neurocomputing</i>. <b>71</b> (<span class="nowrap">16–</span>18): 3150. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.neucom.2008.04.030">10.1016/j.neucom.2008.04.030</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neurocomputing&rft.atitle=Modeling+word+perception+using+the+Elman+network&rft.volume=71&rft.issue=%3Cspan+class%3D%22nowrap%22%3E16%E2%80%93%3C%2Fspan%3E18&rft.pages=3150&rft.date=2008&rft_id=info%3Adoi%2F10.1016%2Fj.neucom.2008.04.030&rft.aulast=Liou&rft.aufirst=Cheng-Yuan&rft.au=Huang%2C+Jau-Chi&rft.au=Yang%2C+Wen-Chie&rft_id=http%3A%2F%2Fntur.lib.ntu.edu.tw%2F%2Fhandle%2F246246%2F155195&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFLiouChengLiouLiou2014" class="citation journal cs1">Liou, Cheng-Yuan; Cheng, Wei-Chen; Liou, Jiun-Wei; Liou, Daw-Ran (2014). "Autoencoder for words". <i>Neurocomputing</i>. <b>139</b>: <span class="nowrap">84–</span>96. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.neucom.2013.09.055">10.1016/j.neucom.2013.09.055</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neurocomputing&rft.atitle=Autoencoder+for+words&rft.volume=139&rft.pages=%3Cspan+class%3D%22nowrap%22%3E84-%3C%2Fspan%3E96&rft.date=2014&rft_id=info%3Adoi%2F10.1016%2Fj.neucom.2013.09.055&rft.aulast=Liou&rft.aufirst=Cheng-Yuan&rft.au=Cheng%2C+Wei-Chen&rft.au=Liou%2C+Jiun-Wei&rft.au=Liou%2C+Daw-Ran&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:12-9"><span class="mw-cite-backlink">^ <a href="#cite_ref-:12_9-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:12_9-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:12_9-2"><sup><i><b>c</b></i></sup></a> <a href="#cite_ref-:12_9-3"><sup><i><b>d</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFKramer1991" class="citation journal cs1">Kramer, Mark A. (1991). <a rel="nofollow" class="external text" href="https://www.researchgate.net/profile/Abir_Alobaid/post/To_learn_a_probability_density_function_by_using_neural_network_can_we_first_estimate_density_using_nonparametric_methods_then_train_the_network/attachment/59d6450279197b80779a031e/AS:451263696510979@1484601057779/download/NL+PCA+by+using+ANN.pdf">"Nonlinear principal component analysis using autoassociative neural networks"</a> <span class="cs1-format">(PDF)</span>. <i>AIChE Journal</i>. <b>37</b> (2): <span class="nowrap">233–</span>243. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1991AIChE..37..233K">1991AIChE..37..233K</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Faic.690370209">10.1002/aic.690370209</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=AIChE+Journal&rft.atitle=Nonlinear+principal+component+analysis+using+autoassociative+neural+networks&rft.volume=37&rft.issue=2&rft.pages=%3Cspan+class%3D%22nowrap%22%3E233-%3C%2Fspan%3E243&rft.date=1991&rft_id=info%3Adoi%2F10.1002%2Faic.690370209&rft_id=info%3Abibcode%2F1991AIChE..37..233K&rft.aulast=Kramer&rft.aufirst=Mark+A.&rft_id=https%3A%2F%2Fwww.researchgate.net%2Fprofile%2FAbir_Alobaid%2Fpost%2FTo_learn_a_probability_density_function_by_using_neural_network_can_we_first_estimate_density_using_nonparametric_methods_then_train_the_network%2Fattachment%2F59d6450279197b80779a031e%2FAS%3A451263696510979%401484601057779%2Fdownload%2FNL%2BPCA%2Bby%2Busing%2BANN.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:7-10"><span class="mw-cite-backlink">^ <a href="#cite_ref-:7_10-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:7_10-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:7_10-2"><sup><i><b>c</b></i></sup></a> <a href="#cite_ref-:7_10-3"><sup><i><b>d</b></i></sup></a> <a href="#cite_ref-:7_10-4"><sup><i><b>e</b></i></sup></a> <a href="#cite_ref-:7_10-5"><sup><i><b>f</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFHintonSalakhutdinov2006" class="citation journal cs1">Hinton, G. E.; Salakhutdinov, R.R. (2006-07-28). "Reducing the Dimensionality of Data with Neural Networks". <i>Science</i>. <b>313</b> (5786): <span class="nowrap">504–</span>507. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2006Sci...313..504H">2006Sci...313..504H</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1126%2Fscience.1127647">10.1126/science.1127647</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/16873662">16873662</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1658773">1658773</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Science&rft.atitle=Reducing+the+Dimensionality+of+Data+with+Neural+Networks&rft.volume=313&rft.issue=5786&rft.pages=%3Cspan+class%3D%22nowrap%22%3E504-%3C%2Fspan%3E507&rft.date=2006-07-28&rft_id=info%3Adoi%2F10.1126%2Fscience.1127647&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1658773%23id-name%3DS2CID&rft_id=info%3Apmid%2F16873662&rft_id=info%3Abibcode%2F2006Sci...313..504H&rft.aulast=Hinton&rft.aufirst=G.+E.&rft.au=Salakhutdinov%2C+R.R.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-bengio-11"><span class="mw-cite-backlink">^ <a href="#cite_ref-bengio_11-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-bengio_11-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBengio2009" class="citation journal cs1">Bengio, Y. (2009). <a rel="nofollow" class="external text" href="http://www.iro.umontreal.ca/~lisa/pointeurs/TR1312.pdf">"Learning Deep Architectures for AI"</a> <span class="cs1-format">(PDF)</span>. <i>Foundations and Trends in Machine Learning</i>. <b>2</b> (8): <span class="nowrap">1795–</span>7. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.701.9550">10.1.1.701.9550</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1561%2F2200000006">10.1561/2200000006</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/23946944">23946944</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:207178999">207178999</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Foundations+and+Trends+in+Machine+Learning&rft.atitle=Learning+Deep+Architectures+for+AI&rft.volume=2&rft.issue=8&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1795-%3C%2Fspan%3E7&rft.date=2009&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.701.9550%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A207178999%23id-name%3DS2CID&rft_id=info%3Apmid%2F23946944&rft_id=info%3Adoi%2F10.1561%2F2200000006&rft.aulast=Bengio&rft.aufirst=Y.&rft_id=http%3A%2F%2Fwww.iro.umontreal.ca%2F~lisa%2Fpointeurs%2FTR1312.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-domingos-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-domingos_12-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFDomingos2015" class="citation book cs1"><a href="/wiki/Pedro_Domingos" title="Pedro Domingos">Domingos, Pedro</a> (2015). "4". <a href="/wiki/The_Master_Algorithm" title="The Master Algorithm"><i>The Master Algorithm: How the Quest for the Ultimate Learning Machine Will Remake Our World</i></a>. Basic Books. "Deeper into the Brain" subsection. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-046506192-1" title="Special:BookSources/978-046506192-1"><bdi>978-046506192-1</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=4&rft.btitle=The+Master+Algorithm%3A+How+the+Quest+for+the+Ultimate+Learning+Machine+Will+Remake+Our+World&rft.pages=%22Deeper+into+the+Brain%22+subsection&rft.pub=Basic+Books&rft.date=2015&rft.isbn=978-046506192-1&rft.aulast=Domingos&rft.aufirst=Pedro&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:1-13"><span class="mw-cite-backlink">^ <a href="#cite_ref-:1_13-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:1_13-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:1_13-2"><sup><i><b>c</b></i></sup></a> <a href="#cite_ref-:1_13-3"><sup><i><b>d</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFMakhzaniFrey2013" class="citation arxiv cs1">Makhzani, Alireza; Frey, Brendan (2013). "K-Sparse Autoencoders". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1312.5663">1312.5663</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=K-Sparse+Autoencoders&rft.date=2013&rft_id=info%3Aarxiv%2F1312.5663&rft.aulast=Makhzani&rft.aufirst=Alireza&rft.au=Frey%2C+Brendan&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:6-14"><span class="mw-cite-backlink">^ <a href="#cite_ref-:6_14-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:6_14-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text">Ng, A. (2011). <a rel="nofollow" class="external text" href="https://web.stanford.edu/class/cs294a/sparseAutoencoder_2011new.pdf">Sparse autoencoder</a>. <i>CS294A Lecture notes</i>, <i>72</i>(2011), 1-19.</span> </li> <li id="cite_note-15"><span class="mw-cite-backlink"><b><a href="#cite_ref-15">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFNairHinton2009" class="citation journal cs1">Nair, Vinod; Hinton, Geoffrey E. (2009). <a rel="nofollow" class="external text" href="http://dl.acm.org/citation.cfm?id=2984093.2984244">"3D Object Recognition with Deep Belief Nets"</a>. <i>Proceedings of the 22nd International Conference on Neural Information Processing Systems</i>. NIPS'09. USA: Curran Associates Inc.: <span class="nowrap">1339–</span>1347. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781615679119" title="Special:BookSources/9781615679119"><bdi>9781615679119</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+the+22nd+International+Conference+on+Neural+Information+Processing+Systems&rft.atitle=3D+Object+Recognition+with+Deep+Belief+Nets&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1339-%3C%2Fspan%3E1347&rft.date=2009&rft.isbn=9781615679119&rft.aulast=Nair&rft.aufirst=Vinod&rft.au=Hinton%2C+Geoffrey+E.&rft_id=http%3A%2F%2Fdl.acm.org%2Fcitation.cfm%3Fid%3D2984093.2984244&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-16"><span class="mw-cite-backlink"><b><a href="#cite_ref-16">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFZengZhangSongLiu2018" class="citation journal cs1">Zeng, Nianyin; Zhang, Hong; Song, Baoye; Liu, Weibo; Li, Yurong; Dobaie, Abdullah M. (2018-01-17). "Facial expression recognition via learning deep sparse autoencoders". <i>Neurocomputing</i>. <b>273</b>: <span class="nowrap">643–</span>649. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.neucom.2017.08.043">10.1016/j.neucom.2017.08.043</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0925-2312">0925-2312</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neurocomputing&rft.atitle=Facial+expression+recognition+via+learning+deep+sparse+autoencoders&rft.volume=273&rft.pages=%3Cspan+class%3D%22nowrap%22%3E643-%3C%2Fspan%3E649&rft.date=2018-01-17&rft_id=info%3Adoi%2F10.1016%2Fj.neucom.2017.08.043&rft.issn=0925-2312&rft.aulast=Zeng&rft.aufirst=Nianyin&rft.au=Zhang%2C+Hong&rft.au=Song%2C+Baoye&rft.au=Liu%2C+Weibo&rft.au=Li%2C+Yurong&rft.au=Dobaie%2C+Abdullah+M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:13-17"><span class="mw-cite-backlink">^ <a href="#cite_ref-:13_17-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:13_17-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFKramer1992" class="citation journal cs1">Kramer, M. A. (1992-04-01). <a rel="nofollow" class="external text" href="https://dx.doi.org/10.1016/0098-1354%2892%2980051-A">"Autoassociative neural networks"</a>. <i>Computers & Chemical Engineering</i>. Neutral network applications in chemical engineering. <b>16</b> (4): <span class="nowrap">313–</span>328. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0098-1354%2892%2980051-A">10.1016/0098-1354(92)80051-A</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0098-1354">0098-1354</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Computers+%26+Chemical+Engineering&rft.atitle=Autoassociative+neural+networks&rft.volume=16&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E313-%3C%2Fspan%3E328&rft.date=1992-04-01&rft_id=info%3Adoi%2F10.1016%2F0098-1354%2892%2980051-A&rft.issn=0098-1354&rft.aulast=Kramer&rft.aufirst=M.+A.&rft_id=https%3A%2F%2Fdx.doi.org%2F10.1016%2F0098-1354%252892%252980051-A&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:5-18"><span class="mw-cite-backlink">^ <a href="#cite_ref-:5_18-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:5_18-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFHintonZemel1993" class="citation journal cs1">Hinton, Geoffrey E; Zemel, Richard (1993). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/1993/hash/9e3cfc48eccf81a0d57663e129aef3cb-Abstract.html">"Autoencoders, Minimum Description Length and Helmholtz Free Energy"</a>. <i>Advances in Neural Information Processing Systems</i>. <b>6</b>. Morgan-Kaufmann.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Advances+in+Neural+Information+Processing+Systems&rft.atitle=Autoencoders%2C+Minimum+Description+Length+and+Helmholtz+Free+Energy&rft.volume=6&rft.date=1993&rft.aulast=Hinton&rft.aufirst=Geoffrey+E&rft.au=Zemel%2C+Richard&rft_id=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper%2F1993%2Fhash%2F9e3cfc48eccf81a0d57663e129aef3cb-Abstract.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-19"><span class="mw-cite-backlink"><b><a href="#cite_ref-19">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFAbidBalinZou2019" class="citation arxiv cs1">Abid, Abubakar; Balin, Muhammad Fatih; Zou, James (2019-01-27). "Concrete Autoencoders for Differentiable Feature Selection and Reconstruction". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1901.09346">1901.09346</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Concrete+Autoencoders+for+Differentiable+Feature+Selection+and+Reconstruction&rft.date=2019-01-27&rft_id=info%3Aarxiv%2F1901.09346&rft.aulast=Abid&rft.aufirst=Abubakar&rft.au=Balin%2C+Muhammad+Fatih&rft.au=Zou%2C+James&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:9-20"><span class="mw-cite-backlink">^ <a href="#cite_ref-:9_20-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:9_20-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:9_20-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFZhouArpitNwoguGovindaraju2014" class="citation arxiv cs1">Zhou, Yingbo; Arpit, Devansh; Nwogu, Ifeoma; Govindaraju, Venu (2014). "Is Joint Training Better for Deep Auto-Encoders?". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1405.1380">1405.1380</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Is+Joint+Training+Better+for+Deep+Auto-Encoders%3F&rft.date=2014&rft_id=info%3Aarxiv%2F1405.1380&rft.aulast=Zhou&rft.aufirst=Yingbo&rft.au=Arpit%2C+Devansh&rft.au=Nwogu%2C+Ifeoma&rft.au=Govindaraju%2C+Venu&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-21"><span class="mw-cite-backlink"><b><a href="#cite_ref-21">^</a></b></span> <span class="reference-text">R. Salakhutdinov and G. E. Hinton, “Deep Boltzmann machines,” in AISTATS, 2009, pp. 448–455.</span> </li> <li id="cite_note-22"><span class="mw-cite-backlink"><b><a href="#cite_ref-22">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFOja1982" class="citation journal cs1">Oja, Erkki (1982-11-01). <a rel="nofollow" class="external text" href="https://link.springer.com/article/10.1007/BF00275687">"Simplified neuron model as a principal component analyzer"</a>. <i>Journal of Mathematical Biology</i>. <b>15</b> (3): <span class="nowrap">267–</span>273. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2FBF00275687">10.1007/BF00275687</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1432-1416">1432-1416</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/7153672">7153672</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Mathematical+Biology&rft.atitle=Simplified+neuron+model+as+a+principal+component+analyzer&rft.volume=15&rft.issue=3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E267-%3C%2Fspan%3E273&rft.date=1982-11-01&rft.issn=1432-1416&rft_id=info%3Apmid%2F7153672&rft_id=info%3Adoi%2F10.1007%2FBF00275687&rft.aulast=Oja&rft.aufirst=Erkki&rft_id=https%3A%2F%2Flink.springer.com%2Farticle%2F10.1007%2FBF00275687&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-auto-23"><span class="mw-cite-backlink">^ <a href="#cite_ref-auto_23-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-auto_23-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBaldiHornik1989" class="citation journal cs1">Baldi, Pierre; Hornik, Kurt (1989-01-01). <a rel="nofollow" class="external text" href="https://www.sciencedirect.com/science/article/abs/pii/0893608089900142">"Neural networks and principal component analysis: Learning from examples without local minima"</a>. <i>Neural Networks</i>. <b>2</b> (1): <span class="nowrap">53–</span>58. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0893-6080%2889%2990014-2">10.1016/0893-6080(89)90014-2</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0893-6080">0893-6080</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Networks&rft.atitle=Neural+networks+and+principal+component+analysis%3A+Learning+from+examples+without+local+minima&rft.volume=2&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E53-%3C%2Fspan%3E58&rft.date=1989-01-01&rft_id=info%3Adoi%2F10.1016%2F0893-6080%2889%2990014-2&rft.issn=0893-6080&rft.aulast=Baldi&rft.aufirst=Pierre&rft.au=Hornik%2C+Kurt&rft_id=https%3A%2F%2Fwww.sciencedirect.com%2Fscience%2Farticle%2Fabs%2Fpii%2F0893608089900142&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-24"><span class="mw-cite-backlink"><b><a href="#cite_ref-24">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFRumelhartMcClellandAU1986" class="citation book cs1">Rumelhart, David E.; McClelland, James L.; AU (1986). "2. A General Framework for Parallel Distributed Processing". <a rel="nofollow" class="external text" href="https://direct.mit.edu/books/book/4424/Parallel-Distributed-ProcessingExplorations-in-the"><i>Parallel Distributed Processing: Explorations in the Microstructure of Cognition: Foundations</i></a>. The MIT Press. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.7551%2Fmitpress%2F5236.001.0001">10.7551/mitpress/5236.001.0001</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-262-29140-8" title="Special:BookSources/978-0-262-29140-8"><bdi>978-0-262-29140-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=2.+A+General+Framework+for+Parallel+Distributed+Processing&rft.btitle=Parallel+Distributed+Processing%3A+Explorations+in+the+Microstructure+of+Cognition%3A+Foundations&rft.pub=The+MIT+Press&rft.date=1986&rft_id=info%3Adoi%2F10.7551%2Fmitpress%2F5236.001.0001&rft.isbn=978-0-262-29140-8&rft.aulast=Rumelhart&rft.aufirst=David+E.&rft.au=McClelland%2C+James+L.&rft.au=AU&rft_id=https%3A%2F%2Fdirect.mit.edu%2Fbooks%2Fbook%2F4424%2FParallel-Distributed-ProcessingExplorations-in-the&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-25"><span class="mw-cite-backlink"><b><a href="#cite_ref-25">^</a></b></span> <span class="reference-text">Harrison TD (1987) A Connectionist framework for continuous speech recognition. Cambridge University Ph. D. dissertation</span> </li> <li id="cite_note-26"><span class="mw-cite-backlink"><b><a href="#cite_ref-26">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFElmanZipser1988" class="citation journal cs1">Elman, Jeffrey L.; Zipser, David (1988-04-01). <a rel="nofollow" class="external text" href="https://pubs.aip.org/jasa/article/83/4/1615/826094/Learning-the-hidden-structure-of-speechLearning">"Learning the hidden structure of speech"</a>. <i>The Journal of the Acoustical Society of America</i>. <b>83</b> (4): <span class="nowrap">1615–</span>1626. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1988ASAJ...83.1615E">1988ASAJ...83.1615E</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1121%2F1.395916">10.1121/1.395916</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0001-4966">0001-4966</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/3372872">3372872</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Journal+of+the+Acoustical+Society+of+America&rft.atitle=Learning+the+hidden+structure+of+speech&rft.volume=83&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1615-%3C%2Fspan%3E1626&rft.date=1988-04-01&rft_id=info%3Adoi%2F10.1121%2F1.395916&rft.issn=0001-4966&rft_id=info%3Apmid%2F3372872&rft_id=info%3Abibcode%2F1988ASAJ...83.1615E&rft.aulast=Elman&rft.aufirst=Jeffrey+L.&rft.au=Zipser%2C+David&rft_id=https%3A%2F%2Fpubs.aip.org%2Fjasa%2Farticle%2F83%2F4%2F1615%2F826094%2FLearning-the-hidden-structure-of-speechLearning&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-27"><span class="mw-cite-backlink"><b><a href="#cite_ref-27">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFCottrellMunroZipser1987" class="citation journal cs1">Cottrell, Garrison W.; Munro, Paul; Zipser, David (1987). <a rel="nofollow" class="external text" href="https://escholarship.org/uc/item/2zs7w6z8">"Learning Internal Representation From Gray-Scale Images: An Example of Extensional Programming"</a>. <i>Proceedings of the Annual Meeting of the Cognitive Science Society</i>. <b>9</b>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+the+Annual+Meeting+of+the+Cognitive+Science+Society&rft.atitle=Learning+Internal+Representation+From+Gray-Scale+Images%3A+An+Example+of+Extensional+Programming&rft.volume=9&rft.date=1987&rft.aulast=Cottrell&rft.aufirst=Garrison+W.&rft.au=Munro%2C+Paul&rft.au=Zipser%2C+David&rft_id=https%3A%2F%2Fescholarship.org%2Fuc%2Fitem%2F2zs7w6z8&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:14-28"><span class="mw-cite-backlink">^ <a href="#cite_ref-:14_28-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:14_28-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBourlardKamp1988" class="citation journal cs1">Bourlard, H.; Kamp, Y. (1988). <a rel="nofollow" class="external text" href="http://infoscience.epfl.ch/record/82601">"Auto-association by multilayer perceptrons and singular value decomposition"</a>. <i>Biological Cybernetics</i>. <b>59</b> (<span class="nowrap">4–</span>5): <span class="nowrap">291–</span>294. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2FBF00332918">10.1007/BF00332918</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/3196773">3196773</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:206775335">206775335</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Biological+Cybernetics&rft.atitle=Auto-association+by+multilayer+perceptrons+and+singular+value+decomposition&rft.volume=59&rft.issue=%3Cspan+class%3D%22nowrap%22%3E4%E2%80%93%3C%2Fspan%3E5&rft.pages=%3Cspan+class%3D%22nowrap%22%3E291-%3C%2Fspan%3E294&rft.date=1988&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A206775335%23id-name%3DS2CID&rft_id=info%3Apmid%2F3196773&rft_id=info%3Adoi%2F10.1007%2FBF00332918&rft.aulast=Bourlard&rft.aufirst=H.&rft.au=Kamp%2C+Y.&rft_id=http%3A%2F%2Finfoscience.epfl.ch%2Frecord%2F82601&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:72-29"><span class="mw-cite-backlink"><b><a href="#cite_ref-:72_29-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFHintonSalakhutdinov2006" class="citation journal cs1">Hinton, G. E.; Salakhutdinov, R.R. (2006-07-28). "Reducing the Dimensionality of Data with Neural Networks". <i>Science</i>. <b>313</b> (5786): <span class="nowrap">504–</span>507. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2006Sci...313..504H">2006Sci...313..504H</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1126%2Fscience.1127647">10.1126/science.1127647</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/16873662">16873662</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1658773">1658773</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Science&rft.atitle=Reducing+the+Dimensionality+of+Data+with+Neural+Networks&rft.volume=313&rft.issue=5786&rft.pages=%3Cspan+class%3D%22nowrap%22%3E504-%3C%2Fspan%3E507&rft.date=2006-07-28&rft_id=info%3Adoi%2F10.1126%2Fscience.1127647&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1658773%23id-name%3DS2CID&rft_id=info%3Apmid%2F16873662&rft_id=info%3Abibcode%2F2006Sci...313..504H&rft.aulast=Hinton&rft.aufirst=G.+E.&rft.au=Salakhutdinov%2C+R.R.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-scholar-30"><span class="mw-cite-backlink"><b><a href="#cite_ref-scholar_30-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFHinton2009" class="citation journal cs1">Hinton G (2009). <a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.5947">"Deep belief networks"</a>. <i>Scholarpedia</i>. <b>4</b> (5): 5947. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2009SchpJ...4.5947H">2009SchpJ...4.5947H</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.4249%2Fscholarpedia.5947">10.4249/scholarpedia.5947</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Scholarpedia&rft.atitle=Deep+belief+networks&rft.volume=4&rft.issue=5&rft.pages=5947&rft.date=2009&rft_id=info%3Adoi%2F10.4249%2Fscholarpedia.5947&rft_id=info%3Abibcode%2F2009SchpJ...4.5947H&rft.aulast=Hinton&rft.aufirst=G&rft_id=https%3A%2F%2Fdoi.org%2F10.4249%252Fscholarpedia.5947&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-31"><span class="mw-cite-backlink"><b><a href="#cite_ref-31">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFSchmidhuber2015" class="citation journal cs1">Schmidhuber, Jürgen (January 2015). "Deep learning in neural networks: An overview". <i>Neural Networks</i>. <b>61</b>: <span class="nowrap">85–</span>117. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1404.7828">1404.7828</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.neunet.2014.09.003">10.1016/j.neunet.2014.09.003</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/25462637">25462637</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:11715509">11715509</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Networks&rft.atitle=Deep+learning+in+neural+networks%3A+An+overview&rft.volume=61&rft.pages=%3Cspan+class%3D%22nowrap%22%3E85-%3C%2Fspan%3E117&rft.date=2015-01&rft_id=info%3Aarxiv%2F1404.7828&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A11715509%23id-name%3DS2CID&rft_id=info%3Apmid%2F25462637&rft_id=info%3Adoi%2F10.1016%2Fj.neunet.2014.09.003&rft.aulast=Schmidhuber&rft.aufirst=J%C3%BCrgen&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-VAE-32"><span class="mw-cite-backlink"><b><a href="#cite_ref-VAE_32-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFDiederik_P_KingmaWelling2013" class="citation arxiv cs1">Diederik P Kingma; Welling, Max (2013). "Auto-Encoding Variational Bayes". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1312.6114">1312.6114</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Auto-Encoding+Variational+Bayes&rft.date=2013&rft_id=info%3Aarxiv%2F1312.6114&rft.au=Diederik+P+Kingma&rft.au=Welling%2C+Max&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-gan_faces-33"><span class="mw-cite-backlink"><b><a href="#cite_ref-gan_faces_33-0">^</a></b></span> <span class="reference-text">Generating Faces with Torch, Boesen A., Larsen L. and Sonderby S.K., 2015 <span class="url"><a rel="nofollow" class="external text" href="http://torch.ch/blog/2015/11/13/gan.html">torch<wbr />.ch<wbr />/blog<wbr />/2015<wbr />/11<wbr />/13<wbr />/gan<wbr />.html</a></span></span> </li> <li id="cite_note-34"><span class="mw-cite-backlink"><b><a href="#cite_ref-34">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFAckleyHintonSejnowski1985" class="citation journal cs1">Ackley, D; Hinton, G; Sejnowski, T (March 1985). <a rel="nofollow" class="external text" href="http://doi.wiley.com/10.1016/S0364-0213(85)80012-4">"A learning algorithm for boltzmann machines"</a>. <i>Cognitive Science</i>. <b>9</b> (1): <span class="nowrap">147–</span>169. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0364-0213%2885%2980012-4">10.1016/S0364-0213(85)80012-4</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Cognitive+Science&rft.atitle=A+learning+algorithm+for+boltzmann+machines&rft.volume=9&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E147-%3C%2Fspan%3E169&rft.date=1985-03&rft_id=info%3Adoi%2F10.1016%2FS0364-0213%2885%2980012-4&rft.aulast=Ackley&rft.aufirst=D&rft.au=Hinton%2C+G&rft.au=Sejnowski%2C+T&rft_id=http%3A%2F%2Fdoi.wiley.com%2F10.1016%2FS0364-0213%2885%2980012-4&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-35"><span class="mw-cite-backlink"><b><a href="#cite_ref-35">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFSchwenkBengio1997" class="citation journal cs1">Schwenk, Holger; Bengio, Yoshua (1997). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/1997/hash/9cb67ffb59554ab1dabb65bcb370ddd9-Abstract.html">"Training Methods for Adaptive Boosting of Neural Networks"</a>. <i>Advances in Neural Information Processing Systems</i>. <b>10</b>. MIT Press.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Advances+in+Neural+Information+Processing+Systems&rft.atitle=Training+Methods+for+Adaptive+Boosting+of+Neural+Networks&rft.volume=10&rft.date=1997&rft.aulast=Schwenk&rft.aufirst=Holger&rft.au=Bengio%2C+Yoshua&rft_id=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper%2F1997%2Fhash%2F9cb67ffb59554ab1dabb65bcb370ddd9-Abstract.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:10-36"><span class="mw-cite-backlink">^ <a href="#cite_ref-:10_36-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:10_36-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://github.com/zalandoresearch/fashion-mnist">"Fashion MNIST"</a>. <i><a href="/wiki/GitHub" title="GitHub">GitHub</a></i>. 2019-07-12.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=GitHub&rft.atitle=Fashion+MNIST&rft.date=2019-07-12&rft_id=https%3A%2F%2Fgithub.com%2Fzalandoresearch%2Ffashion-mnist&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:3-37"><span class="mw-cite-backlink">^ <a href="#cite_ref-:3_37-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:3_37-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFSalakhutdinovHinton2009" class="citation journal cs1">Salakhutdinov, Ruslan; Hinton, Geoffrey (2009-07-01). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ijar.2008.11.006">"Semantic hashing"</a>. <i>International Journal of Approximate Reasoning</i>. Special Section on Graphical Models and Information Retrieval. <b>50</b> (7): <span class="nowrap">969–</span>978. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ijar.2008.11.006">10.1016/j.ijar.2008.11.006</a></span>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0888-613X">0888-613X</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=International+Journal+of+Approximate+Reasoning&rft.atitle=Semantic+hashing&rft.volume=50&rft.issue=7&rft.pages=%3Cspan+class%3D%22nowrap%22%3E969-%3C%2Fspan%3E978&rft.date=2009-07-01&rft_id=info%3Adoi%2F10.1016%2Fj.ijar.2008.11.006&rft.issn=0888-613X&rft.aulast=Salakhutdinov&rft.aufirst=Ruslan&rft.au=Hinton%2C+Geoffrey&rft_id=https%3A%2F%2Fdoi.org%2F10.1016%252Fj.ijar.2008.11.006&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-38"><span class="mw-cite-backlink"><b><a href="#cite_ref-38">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFChiccoSadowskiBaldi2014" class="citation book cs1">Chicco, Davide; Sadowski, Peter; Baldi, Pierre (2014). "Deep autoencoder neural networks for gene ontology annotation predictions". <a rel="nofollow" class="external text" href="http://dl.acm.org/citation.cfm?id=2649442"><i>Proceedings of the 5th ACM Conference on Bioinformatics, Computational Biology, and Health Informatics - BCB '14</i></a>. p. 533. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F2649387.2649442">10.1145/2649387.2649442</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/11311%2F964622">11311/964622</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781450328944" title="Special:BookSources/9781450328944"><bdi>9781450328944</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:207217210">207217210</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Deep+autoencoder+neural+networks+for+gene+ontology+annotation+predictions&rft.btitle=Proceedings+of+the+5th+ACM+Conference+on+Bioinformatics%2C+Computational+Biology%2C+and+Health+Informatics+-+BCB+%2714&rft.pages=533&rft.date=2014&rft_id=info%3Ahdl%2F11311%2F964622&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A207217210%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F2649387.2649442&rft.isbn=9781450328944&rft.aulast=Chicco&rft.aufirst=Davide&rft.au=Sadowski%2C+Peter&rft.au=Baldi%2C+Pierre&rft_id=http%3A%2F%2Fdl.acm.org%2Fcitation.cfm%3Fid%3D2649442&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-39"><span class="mw-cite-backlink"><b><a href="#cite_ref-39">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFPlaut2018" class="citation arxiv cs1">Plaut, E (2018). "From Principal Subspaces to Principal Components with Linear Autoencoders". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1804.10253">1804.10253</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=From+Principal+Subspaces+to+Principal+Components+with+Linear+Autoencoders&rft.date=2018&rft_id=info%3Aarxiv%2F1804.10253&rft.aulast=Plaut&rft.aufirst=E&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-40"><span class="mw-cite-backlink"><b><a href="#cite_ref-40">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFMorales-ForeroBassetto2019" class="citation book cs1">Morales-Forero, A.; Bassetto, S. (December 2019). <a rel="nofollow" class="external text" href="https://ieeexplore.ieee.org/document/8978509">"Case Study: A Semi-Supervised Methodology for Anomaly Detection and Diagnosis"</a>. <i>2019 IEEE International Conference on Industrial Engineering and Engineering Management (IEEM)</i>. Macao, Macao: IEEE. pp. <span class="nowrap">1031–</span>1037. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FIEEM44572.2019.8978509">10.1109/IEEM44572.2019.8978509</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-7281-3804-6" title="Special:BookSources/978-1-7281-3804-6"><bdi>978-1-7281-3804-6</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:211027131">211027131</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Case+Study%3A+A+Semi-Supervised+Methodology+for+Anomaly+Detection+and+Diagnosis&rft.btitle=2019+IEEE+International+Conference+on+Industrial+Engineering+and+Engineering+Management+%28IEEM%29&rft.place=Macao%2C+Macao&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1031-%3C%2Fspan%3E1037&rft.pub=IEEE&rft.date=2019-12&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A211027131%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FIEEM44572.2019.8978509&rft.isbn=978-1-7281-3804-6&rft.aulast=Morales-Forero&rft.aufirst=A.&rft.au=Bassetto%2C+S.&rft_id=https%3A%2F%2Fieeexplore.ieee.org%2Fdocument%2F8978509&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-41"><span class="mw-cite-backlink"><b><a href="#cite_ref-41">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFSakuradaYairi2014" class="citation book cs1">Sakurada, Mayu; Yairi, Takehisa (December 2014). <a rel="nofollow" class="external text" href="http://dl.acm.org/citation.cfm?doid=2689746.2689747">"Anomaly Detection Using Autoencoders with Nonlinear Dimensionality Reduction"</a>. <i>Proceedings of the MLSDA 2014 2nd Workshop on Machine Learning for Sensory Data Analysis</i>. Gold Coast, Australia QLD, Australia: ACM Press. pp. <span class="nowrap">4–</span>11. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F2689746.2689747">10.1145/2689746.2689747</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-4503-3159-3" title="Special:BookSources/978-1-4503-3159-3"><bdi>978-1-4503-3159-3</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:14613395">14613395</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Anomaly+Detection+Using+Autoencoders+with+Nonlinear+Dimensionality+Reduction&rft.btitle=Proceedings+of+the+MLSDA+2014+2nd+Workshop+on+Machine+Learning+for+Sensory+Data+Analysis&rft.place=Gold+Coast%2C+Australia+QLD%2C+Australia&rft.pages=%3Cspan+class%3D%22nowrap%22%3E4-%3C%2Fspan%3E11&rft.pub=ACM+Press&rft.date=2014-12&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A14613395%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F2689746.2689747&rft.isbn=978-1-4503-3159-3&rft.aulast=Sakurada&rft.aufirst=Mayu&rft.au=Yairi%2C+Takehisa&rft_id=http%3A%2F%2Fdl.acm.org%2Fcitation.cfm%3Fdoid%3D2689746.2689747&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-:8-42"><span class="mw-cite-backlink">^ <a href="#cite_ref-:8_42-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:8_42-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:8_42-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text">An, J., & Cho, S. (2015). <a rel="nofollow" class="external text" href="http://dm.snu.ac.kr/static/docs/TR/SNUDM-TR-2015-03.pdf">Variational Autoencoder based Anomaly Detection using Reconstruction Probability</a>. <i>Special Lecture on IE</i>, <i>2</i>, 1-18.</span> </li> <li id="cite_note-43"><span class="mw-cite-backlink"><b><a href="#cite_ref-43">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFZhouPaffenroth2017" class="citation book cs1">Zhou, Chong; Paffenroth, Randy C. (2017-08-04). <a rel="nofollow" class="external text" href="https://dl.acm.org/doi/10.1145/3097983.3098052">"Anomaly Detection with Robust Deep Autoencoders"</a>. <i>Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining</i>. ACM. pp. <span class="nowrap">665–</span>674. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F3097983.3098052">10.1145/3097983.3098052</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-4503-4887-4" title="Special:BookSources/978-1-4503-4887-4"><bdi>978-1-4503-4887-4</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:207557733">207557733</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Anomaly+Detection+with+Robust+Deep+Autoencoders&rft.btitle=Proceedings+of+the+23rd+ACM+SIGKDD+International+Conference+on+Knowledge+Discovery+and+Data+Mining&rft.pages=%3Cspan+class%3D%22nowrap%22%3E665-%3C%2Fspan%3E674&rft.pub=ACM&rft.date=2017-08-04&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A207557733%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F3097983.3098052&rft.isbn=978-1-4503-4887-4&rft.aulast=Zhou&rft.aufirst=Chong&rft.au=Paffenroth%2C+Randy+C.&rft_id=https%3A%2F%2Fdl.acm.org%2Fdoi%2F10.1145%2F3097983.3098052&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-44"><span class="mw-cite-backlink"><b><a href="#cite_ref-44">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFRibeiroLazzarettiLopes2018" class="citation journal cs1">Ribeiro, Manassés; Lazzaretti, André Eugênio; Lopes, Heitor Silvério (2018). "A study of deep convolutional auto-encoders for anomaly detection in videos". <i>Pattern Recognition Letters</i>. <b>105</b>: <span class="nowrap">13–</span>22. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2018PaReL.105...13R">2018PaReL.105...13R</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.patrec.2017.07.016">10.1016/j.patrec.2017.07.016</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Pattern+Recognition+Letters&rft.atitle=A+study+of+deep+convolutional+auto-encoders+for+anomaly+detection+in+videos&rft.volume=105&rft.pages=%3Cspan+class%3D%22nowrap%22%3E13-%3C%2Fspan%3E22&rft.date=2018&rft_id=info%3Adoi%2F10.1016%2Fj.patrec.2017.07.016&rft_id=info%3Abibcode%2F2018PaReL.105...13R&rft.aulast=Ribeiro&rft.aufirst=Manass%C3%A9s&rft.au=Lazzaretti%2C+Andr%C3%A9+Eug%C3%AAnio&rft.au=Lopes%2C+Heitor+Silv%C3%A9rio&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-45"><span class="mw-cite-backlink"><b><a href="#cite_ref-45">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFNalisnickMatsukawaTehGorur2019" class="citation arxiv cs1">Nalisnick, Eric; Matsukawa, Akihiro; Teh, Yee Whye; Gorur, Dilan; Lakshminarayanan, Balaji (2019-02-24). "Do Deep Generative Models Know What They Don't Know?". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1810.09136">1810.09136</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Do+Deep+Generative+Models+Know+What+They+Don%27t+Know%3F&rft.date=2019-02-24&rft_id=info%3Aarxiv%2F1810.09136&rft.aulast=Nalisnick&rft.aufirst=Eric&rft.au=Matsukawa%2C+Akihiro&rft.au=Teh%2C+Yee+Whye&rft.au=Gorur%2C+Dilan&rft.au=Lakshminarayanan%2C+Balaji&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-46"><span class="mw-cite-backlink"><b><a href="#cite_ref-46">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFXiaoYanAmit2020" class="citation journal cs1">Xiao, Zhisheng; Yan, Qing; Amit, Yali (2020). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/2020/hash/eddea82ad2755b24c4e168c5fc2ebd40-Abstract.html">"Likelihood Regret: An Out-of-Distribution Detection Score For Variational Auto-encoder"</a>. <i>Advances in Neural Information Processing Systems</i>. <b>33</b>. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2003.02977">2003.02977</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Advances+in+Neural+Information+Processing+Systems&rft.atitle=Likelihood+Regret%3A+An+Out-of-Distribution+Detection+Score+For+Variational+Auto-encoder&rft.volume=33&rft.date=2020&rft_id=info%3Aarxiv%2F2003.02977&rft.aulast=Xiao&rft.aufirst=Zhisheng&rft.au=Yan%2C+Qing&rft.au=Amit%2C+Yali&rft_id=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper%2F2020%2Fhash%2Feddea82ad2755b24c4e168c5fc2ebd40-Abstract.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-47"><span class="mw-cite-backlink"><b><a href="#cite_ref-47">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFTheisShiCunninghamHuszár2017" class="citation arxiv cs1">Theis, Lucas; Shi, Wenzhe; Cunningham, Andrew; Huszár, Ferenc (2017). "Lossy Image Compression with Compressive Autoencoders". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1703.00395">1703.00395</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Lossy+Image+Compression+with+Compressive+Autoencoders&rft.date=2017&rft_id=info%3Aarxiv%2F1703.00395&rft.aulast=Theis&rft.aufirst=Lucas&rft.au=Shi%2C+Wenzhe&rft.au=Cunningham%2C+Andrew&rft.au=Husz%C3%A1r%2C+Ferenc&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-48"><span class="mw-cite-backlink"><b><a href="#cite_ref-48">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBalleLaparraSimoncelli2017" class="citation book cs1">Balle, J; Laparra, V; Simoncelli, EP (April 2017). "End-to-end optimized image compression". <i>International Conference on Learning Representations</i>. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1611.01704">1611.01704</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=End-to-end+optimized+image+compression&rft.btitle=International+Conference+on+Learning+Representations&rft.date=2017-04&rft_id=info%3Aarxiv%2F1611.01704&rft.aulast=Balle&rft.aufirst=J&rft.au=Laparra%2C+V&rft.au=Simoncelli%2C+EP&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-49"><span class="mw-cite-backlink"><b><a href="#cite_ref-49">^</a></b></span> <span class="reference-text">Cho, K. (2013, February). Simple sparsification improves sparse denoising autoencoders in denoising highly corrupted images. In <i>International Conference on Machine Learning</i> (pp. 432-440).</span> </li> <li id="cite_note-50"><span class="mw-cite-backlink"><b><a href="#cite_ref-50">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFCho2013" class="citation arxiv cs1">Cho, Kyunghyun (2013). "Boltzmann Machines and Denoising Autoencoders for Image Denoising". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1301.3468">1301.3468</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Boltzmann+Machines+and+Denoising+Autoencoders+for+Image+Denoising&rft.date=2013&rft_id=info%3Aarxiv%2F1301.3468&rft.aulast=Cho&rft.aufirst=Kyunghyun&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-51"><span class="mw-cite-backlink"><b><a href="#cite_ref-51">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBuadesCollMorel2005" class="citation journal cs1">Buades, A.; Coll, B.; Morel, J. M. (2005). <a rel="nofollow" class="external text" href="https://hal.archives-ouvertes.fr/hal-00271141">"A Review of Image Denoising Algorithms, with a New One"</a>. <i>Multiscale Modeling & Simulation</i>. <b>4</b> (2): <span class="nowrap">490–</span>530. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1137%2F040616024">10.1137/040616024</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:218466166">218466166</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Multiscale+Modeling+%26+Simulation&rft.atitle=A+Review+of+Image+Denoising+Algorithms%2C+with+a+New+One&rft.volume=4&rft.issue=2&rft.pages=%3Cspan+class%3D%22nowrap%22%3E490-%3C%2Fspan%3E530&rft.date=2005&rft_id=info%3Adoi%2F10.1137%2F040616024&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A218466166%23id-name%3DS2CID&rft.aulast=Buades&rft.aufirst=A.&rft.au=Coll%2C+B.&rft.au=Morel%2C+J.+M.&rft_id=https%3A%2F%2Fhal.archives-ouvertes.fr%2Fhal-00271141&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-52"><span class="mw-cite-backlink"><b><a href="#cite_ref-52">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGondara2016" class="citation book cs1">Gondara, Lovedeep (December 2016). "Medical Image Denoising Using Convolutional Denoising Autoencoders". <i>2016 IEEE 16th International Conference on Data Mining Workshops (ICDMW)</i>. Barcelona, Spain: IEEE. pp. <span class="nowrap">241–</span>246. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1608.04667">1608.04667</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2016arXiv160804667G">2016arXiv160804667G</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FICDMW.2016.0041">10.1109/ICDMW.2016.0041</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781509059102" title="Special:BookSources/9781509059102"><bdi>9781509059102</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:14354973">14354973</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Medical+Image+Denoising+Using+Convolutional+Denoising+Autoencoders&rft.btitle=2016+IEEE+16th+International+Conference+on+Data+Mining+Workshops+%28ICDMW%29&rft.place=Barcelona%2C+Spain&rft.pages=%3Cspan+class%3D%22nowrap%22%3E241-%3C%2Fspan%3E246&rft.pub=IEEE&rft.date=2016-12&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A14354973%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2016arXiv160804667G&rft_id=info%3Aarxiv%2F1608.04667&rft_id=info%3Adoi%2F10.1109%2FICDMW.2016.0041&rft.isbn=9781509059102&rft.aulast=Gondara&rft.aufirst=Lovedeep&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-53"><span class="mw-cite-backlink"><b><a href="#cite_ref-53">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFZengYuWangLi2017" class="citation journal cs1">Zeng, Kun; Yu, Jun; Wang, Ruxin; Li, Cuihua; Tao, Dacheng (January 2017). "Coupled Deep Autoencoder for Single Image Super-Resolution". <i>IEEE Transactions on Cybernetics</i>. <b>47</b> (1): <span class="nowrap">27–</span>37. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTCYB.2015.2501373">10.1109/TCYB.2015.2501373</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2168-2267">2168-2267</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/26625442">26625442</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:20787612">20787612</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Cybernetics&rft.atitle=Coupled+Deep+Autoencoder+for+Single+Image+Super-Resolution&rft.volume=47&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E27-%3C%2Fspan%3E37&rft.date=2017-01&rft.issn=2168-2267&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A20787612%23id-name%3DS2CID&rft_id=info%3Apmid%2F26625442&rft_id=info%3Adoi%2F10.1109%2FTCYB.2015.2501373&rft.aulast=Zeng&rft.aufirst=Kun&rft.au=Yu%2C+Jun&rft.au=Wang%2C+Ruxin&rft.au=Li%2C+Cuihua&rft.au=Tao%2C+Dacheng&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-54"><span class="mw-cite-backlink"><b><a href="#cite_ref-54">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFTzu-HsiSanchezHeshamNasir_M.2017" class="citation book cs1">Tzu-Hsi, Song; Sanchez, Victor; Hesham, EIDaly; Nasir M., Rajpoot (2017). "Hybrid deep autoencoder with Curvature Gaussian for detection of various types of cells in bone marrow trephine biopsy images". <i>2017 IEEE 14th International Symposium on Biomedical Imaging (ISBI 2017)</i>. pp. <span class="nowrap">1040–</span>1043. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FISBI.2017.7950694">10.1109/ISBI.2017.7950694</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-5090-1172-8" title="Special:BookSources/978-1-5090-1172-8"><bdi>978-1-5090-1172-8</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:7433130">7433130</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Hybrid+deep+autoencoder+with+Curvature+Gaussian+for+detection+of+various+types+of+cells+in+bone+marrow+trephine+biopsy+images&rft.btitle=2017+IEEE+14th+International+Symposium+on+Biomedical+Imaging+%28ISBI+2017%29&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1040-%3C%2Fspan%3E1043&rft.date=2017&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A7433130%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FISBI.2017.7950694&rft.isbn=978-1-5090-1172-8&rft.aulast=Tzu-Hsi&rft.aufirst=Song&rft.au=Sanchez%2C+Victor&rft.au=Hesham%2C+EIDaly&rft.au=Nasir+M.%2C+Rajpoot&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-55"><span class="mw-cite-backlink"><b><a href="#cite_ref-55">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFXuXiangLiuGilmore2016" class="citation journal cs1">Xu, Jun; Xiang, Lei; Liu, Qingshan; Gilmore, Hannah; Wu, Jianzhong; Tang, Jinghai; Madabhushi, Anant (January 2016). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4729702">"Stacked Sparse Autoencoder (SSAE) for Nuclei Detection on Breast Cancer Histopathology Images"</a>. <i>IEEE Transactions on Medical Imaging</i>. <b>35</b> (1): <span class="nowrap">119–</span>130. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTMI.2015.2458702">10.1109/TMI.2015.2458702</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4729702">4729702</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/26208307">26208307</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Medical+Imaging&rft.atitle=Stacked+Sparse+Autoencoder+%28SSAE%29+for+Nuclei+Detection+on+Breast+Cancer+Histopathology+Images&rft.volume=35&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E119-%3C%2Fspan%3E130&rft.date=2016-01&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC4729702%23id-name%3DPMC&rft_id=info%3Apmid%2F26208307&rft_id=info%3Adoi%2F10.1109%2FTMI.2015.2458702&rft.aulast=Xu&rft.aufirst=Jun&rft.au=Xiang%2C+Lei&rft.au=Liu%2C+Qingshan&rft.au=Gilmore%2C+Hannah&rft.au=Wu%2C+Jianzhong&rft.au=Tang%2C+Jinghai&rft.au=Madabhushi%2C+Anant&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC4729702&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-56"><span class="mw-cite-backlink"><b><a href="#cite_ref-56">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFMartinez-MurciaOrtizGorrizRamirez2020" class="citation journal cs1">Martinez-Murcia, Francisco J.; Ortiz, Andres; Gorriz, Juan M.; Ramirez, Javier; Castillo-Barnes, Diego (2020). <a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FJBHI.2019.2914970">"Studying the Manifold Structure of Alzheimer's Disease: A Deep Learning Approach Using Convolutional Autoencoders"</a>. <i>IEEE Journal of Biomedical and Health Informatics</i>. <b>24</b> (1): <span class="nowrap">17–</span>26. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FJBHI.2019.2914970">10.1109/JBHI.2019.2914970</a></span>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/10630%2F28806">10630/28806</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/31217131">31217131</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:195187846">195187846</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Journal+of+Biomedical+and+Health+Informatics&rft.atitle=Studying+the+Manifold+Structure+of+Alzheimer%27s+Disease%3A+A+Deep+Learning+Approach+Using+Convolutional+Autoencoders&rft.volume=24&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E17-%3C%2Fspan%3E26&rft.date=2020&rft_id=info%3Ahdl%2F10630%2F28806&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A195187846%23id-name%3DS2CID&rft_id=info%3Apmid%2F31217131&rft_id=info%3Adoi%2F10.1109%2FJBHI.2019.2914970&rft.aulast=Martinez-Murcia&rft.aufirst=Francisco+J.&rft.au=Ortiz%2C+Andres&rft.au=Gorriz%2C+Juan+M.&rft.au=Ramirez%2C+Javier&rft.au=Castillo-Barnes%2C+Diego&rft_id=https%3A%2F%2Fdoi.org%2F10.1109%252FJBHI.2019.2914970&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-57"><span class="mw-cite-backlink"><b><a href="#cite_ref-57">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFZhavoronkov2019" class="citation journal cs1">Zhavoronkov, Alex (2019). "Deep learning enables rapid identification of potent DDR1 kinase inhibitors". <i>Nature Biotechnology</i>. <b>37</b> (9): <span class="nowrap">1038–</span>1040. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fs41587-019-0224-x">10.1038/s41587-019-0224-x</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/31477924">31477924</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:201716327">201716327</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Nature+Biotechnology&rft.atitle=Deep+learning+enables+rapid+identification+of+potent+DDR1+kinase+inhibitors&rft.volume=37&rft.issue=9&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1038-%3C%2Fspan%3E1040&rft.date=2019&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A201716327%23id-name%3DS2CID&rft_id=info%3Apmid%2F31477924&rft_id=info%3Adoi%2F10.1038%2Fs41587-019-0224-x&rft.aulast=Zhavoronkov&rft.aufirst=Alex&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-58"><span class="mw-cite-backlink"><b><a href="#cite_ref-58">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGregory" class="citation magazine cs1">Gregory, Barber. <a rel="nofollow" class="external text" href="https://www.wired.com/story/molecule-designed-ai-exhibits-druglike-qualities/">"A Molecule Designed By AI Exhibits 'Druglike' Qualities"</a>. <i>Wired</i>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Wired&rft.atitle=A+Molecule+Designed+By+AI+Exhibits+%27Druglike%27+Qualities&rft.aulast=Gregory&rft.aufirst=Barber&rft_id=https%3A%2F%2Fwww.wired.com%2Fstory%2Fmolecule-designed-ai-exhibits-druglike-qualities%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-59"><span class="mw-cite-backlink"><b><a href="#cite_ref-59">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFDeMaityGoelShitole2017" class="citation book cs1">De, Shaunak; Maity, Abhishek; Goel, Vritti; Shitole, Sanjay; Bhattacharya, Avik (2017). "Predicting the popularity of instagram posts for a lifestyle magazine using deep learning". <i>2017 2nd IEEE International Conference on Communication Systems, Computing and IT Applications (CSCITA)</i>. pp. <span class="nowrap">174–</span>177. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FCSCITA.2017.8066548">10.1109/CSCITA.2017.8066548</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-5090-4381-1" title="Special:BookSources/978-1-5090-4381-1"><bdi>978-1-5090-4381-1</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:35350962">35350962</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Predicting+the+popularity+of+instagram+posts+for+a+lifestyle+magazine+using+deep+learning&rft.btitle=2017+2nd+IEEE+International+Conference+on+Communication+Systems%2C+Computing+and+IT+Applications+%28CSCITA%29&rft.pages=%3Cspan+class%3D%22nowrap%22%3E174-%3C%2Fspan%3E177&rft.date=2017&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A35350962%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FCSCITA.2017.8066548&rft.isbn=978-1-5090-4381-1&rft.aulast=De&rft.aufirst=Shaunak&rft.au=Maity%2C+Abhishek&rft.au=Goel%2C+Vritti&rft.au=Shitole%2C+Sanjay&rft.au=Bhattacharya%2C+Avik&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-60"><span class="mw-cite-backlink"><b><a href="#cite_ref-60">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFChoBart_van_MerrienboerBahdanauBengio2014" class="citation arxiv cs1">Cho, Kyunghyun; Bart van Merrienboer; Bahdanau, Dzmitry; Bengio, Yoshua (2014). "On the Properties of Neural Machine Translation: Encoder-Decoder Approaches". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1409.1259">1409.1259</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=On+the+Properties+of+Neural+Machine+Translation%3A+Encoder-Decoder+Approaches&rft.date=2014&rft_id=info%3Aarxiv%2F1409.1259&rft.aulast=Cho&rft.aufirst=Kyunghyun&rft.au=Bart+van+Merrienboer&rft.au=Bahdanau%2C+Dzmitry&rft.au=Bengio%2C+Yoshua&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-61"><span class="mw-cite-backlink"><b><a href="#cite_ref-61">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFSutskeverVinyalsLe2014" class="citation arxiv cs1">Sutskever, Ilya; Vinyals, Oriol; Le, Quoc V. (2014). "Sequence to Sequence Learning with Neural Networks". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1409.3215">1409.3215</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Sequence+to+Sequence+Learning+with+Neural+Networks&rft.date=2014&rft_id=info%3Aarxiv%2F1409.3215&rft.aulast=Sutskever&rft.aufirst=Ilya&rft.au=Vinyals%2C+Oriol&rft.au=Le%2C+Quoc+V.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-62"><span class="mw-cite-backlink"><b><a href="#cite_ref-62">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFHanKuang2018" class="citation arxiv cs1">Han, Lifeng; Kuang, Shaohui (2018). "Incorporating Chinese Radicals into Neural Machine Translation: Deeper Than Character Level". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.01565">1805.01565</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CL">cs.CL</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=Incorporating+Chinese+Radicals+into+Neural+Machine+Translation%3A+Deeper+Than+Character+Level&rft.date=2018&rft_id=info%3Aarxiv%2F1805.01565&rft.aulast=Han&rft.aufirst=Lifeng&rft.au=Kuang%2C+Shaohui&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> <li id="cite_note-63"><span class="mw-cite-backlink"><b><a href="#cite_ref-63">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFAlnaseriAlzubaidiHimeurTimmermann2024" class="citation arxiv cs1">Alnaseri, Omar; Alzubaidi, Laith; Himeur, Yassine; Timmermann, Jens (2024). "A Review on Deep Learning Autoencoder in the Design of Next-Generation Communication Systems". <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2412.13843">2412.13843</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/eess.SP">eess.SP</a>].</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=preprint&rft.jtitle=arXiv&rft.atitle=A+Review+on+Deep+Learning+Autoencoder+in+the+Design+of+Next-Generation+Communication+Systems&rft.date=2024&rft_id=info%3Aarxiv%2F2412.13843&rft.aulast=Alnaseri&rft.aufirst=Omar&rft.au=Alzubaidi%2C+Laith&rft.au=Himeur%2C+Yassine&rft.au=Timmermann%2C+Jens&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutoencoder" class="Z3988"></span></span> </li> </ol></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="Artificial_intelligence_(AI)752" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1239400231" /><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Artificial_intelligence_navbox" title="Template:Artificial intelligence navbox"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Artificial_intelligence_navbox" title="Template talk:Artificial intelligence navbox"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Artificial_intelligence_navbox" title="Special:EditPage/Template:Artificial intelligence navbox"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Artificial_intelligence_(AI)752" style="font-size:114%;margin:0 4em"><a href="/wiki/Artificial_intelligence" title="Artificial intelligence">Artificial intelligence</a> (AI)</div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div><a href="/wiki/History_of_artificial_intelligence" title="History of artificial intelligence">History</a> (<a href="/wiki/Timeline_of_artificial_intelligence" title="Timeline of artificial intelligence">timeline</a>)</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Parameter" title="Parameter">Parameter</a> <ul><li><a href="/wiki/Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">Hyperparameter</a></li></ul></li> <li><a href="/wiki/Loss_functions_for_classification" title="Loss functions for classification">Loss functions</a></li> <li><a href="/wiki/Regression_analysis" title="Regression analysis">Regression</a> <ul><li><a href="/wiki/Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li> <li><a href="/wiki/Double_descent" title="Double descent">Double descent</a></li> <li><a href="/wiki/Overfitting" title="Overfitting">Overfitting</a></li></ul></li> <li><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></li> <li><a href="/wiki/Gradient_descent" title="Gradient descent">Gradient descent</a> <ul><li><a href="/wiki/Stochastic_gradient_descent" title="Stochastic gradient descent">SGD</a></li> <li><a href="/wiki/Quasi-Newton_method" title="Quasi-Newton method">Quasi-Newton method</a></li> <li><a href="/wiki/Conjugate_gradient_method" title="Conjugate gradient method">Conjugate gradient method</a></li></ul></li> <li><a href="/wiki/Backpropagation" title="Backpropagation">Backpropagation</a></li> <li><a href="/wiki/Attention_(machine_learning)" title="Attention (machine learning)">Attention</a></li> <li><a href="/wiki/Convolution" title="Convolution">Convolution</a></li> <li><a href="/wiki/Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization</a> <ul><li><a href="/wiki/Batch_normalization" title="Batch normalization">Batchnorm</a></li></ul></li> <li><a href="/wiki/Activation_function" title="Activation function">Activation</a> <ul><li><a href="/wiki/Softmax_function" title="Softmax function">Softmax</a></li> <li><a href="/wiki/Sigmoid_function" title="Sigmoid function">Sigmoid</a></li> <li><a href="/wiki/Rectifier_(neural_networks)" title="Rectifier (neural networks)">Rectifier</a></li></ul></li> <li><a href="/wiki/Gating_mechanism" title="Gating mechanism">Gating</a></li> <li><a href="/wiki/Weight_initialization" title="Weight initialization">Weight initialization</a></li> <li><a href="/wiki/Regularization_(mathematics)" title="Regularization (mathematics)">Regularization</a></li> <li><a href="/wiki/Training,_validation,_and_test_data_sets" title="Training, validation, and test data sets">Datasets</a> <ul><li><a href="/wiki/Data_augmentation" title="Data augmentation">Augmentation</a></li></ul></li> <li><a href="/wiki/Prompt_engineering" title="Prompt engineering">Prompt engineering</a></li> <li><a href="/wiki/Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a> <ul><li><a href="/wiki/Q-learning" title="Q-learning">Q-learning</a></li> <li><a href="/wiki/State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li> <li><a href="/wiki/Imitation_learning" title="Imitation learning">Imitation</a></li> <li><a href="/wiki/Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li></ul></li> <li><a href="/wiki/Diffusion_process" title="Diffusion process">Diffusion</a></li> <li><a href="/wiki/Latent_diffusion_model" title="Latent diffusion model">Latent diffusion model</a></li> <li><a href="/wiki/Autoregressive_model" title="Autoregressive model">Autoregression</a></li> <li><a href="/wiki/Adversarial_machine_learning" title="Adversarial machine learning">Adversary</a></li> <li><a href="/wiki/Retrieval-augmented_generation" title="Retrieval-augmented generation">RAG</a></li> <li><a href="/wiki/Uncanny_valley" title="Uncanny valley">Uncanny valley</a></li> <li><a href="/wiki/Reinforcement_learning_from_human_feedback" title="Reinforcement learning from human feedback">RLHF</a></li> <li><a href="/wiki/Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li> <li><a href="/wiki/Recursive_self-improvement" title="Recursive self-improvement">Recursive self-improvement</a></li> <li><a href="/wiki/Word_embedding" title="Word embedding">Word embedding</a></li> <li><a href="/wiki/Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Applications</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Machine_learning" title="Machine learning">Machine learning</a> <ul><li><a href="/wiki/Prompt_engineering#In-context_learning" title="Prompt engineering">In-context learning</a></li></ul></li> <li><a href="/wiki/Neural_network_(machine_learning)" title="Neural network (machine learning)">Artificial neural network</a> <ul><li><a href="/wiki/Deep_learning" title="Deep learning">Deep learning</a></li></ul></li> <li><a href="/wiki/Language_model" title="Language model">Language model</a> <ul><li><a href="/wiki/Large_language_model" title="Large language model">Large language model</a></li> <li><a href="/wiki/Neural_machine_translation" title="Neural machine translation">NMT</a></li></ul></li> <li><a href="/wiki/Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Implementations</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Audio–visual</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/AlexNet" title="AlexNet">AlexNet</a></li> <li><a href="/wiki/WaveNet" title="WaveNet">WaveNet</a></li> <li><a href="/wiki/Human_image_synthesis" title="Human image synthesis">Human image synthesis</a></li> <li><a href="/wiki/Handwriting_recognition" title="Handwriting recognition">HWR</a></li> <li><a href="/wiki/Optical_character_recognition" title="Optical character recognition">OCR</a></li> <li><a href="/wiki/Deep_learning_speech_synthesis" title="Deep learning speech synthesis">Speech synthesis</a> <ul><li><a href="/wiki/15.ai" title="15.ai">15.ai</a></li> <li><a href="/wiki/ElevenLabs" title="ElevenLabs">ElevenLabs</a></li></ul></li> <li><a href="/wiki/Speech_recognition" title="Speech recognition">Speech recognition</a> <ul><li><a href="/wiki/Whisper_(speech_recognition_system)" title="Whisper (speech recognition system)">Whisper</a></li></ul></li> <li><a href="/wiki/Facial_recognition_system" title="Facial recognition system">Facial recognition</a></li> <li><a href="/wiki/AlphaFold" title="AlphaFold">AlphaFold</a></li> <li><a href="/wiki/Text-to-image_model" title="Text-to-image model">Text-to-image models</a> <ul><li><a href="/wiki/Aurora_(text-to-image_model)" class="mw-redirect" title="Aurora (text-to-image model)">Aurora</a></li> <li><a href="/wiki/DALL-E" title="DALL-E">DALL-E</a></li> <li><a href="/wiki/Adobe_Firefly" title="Adobe Firefly">Firefly</a></li> <li><a href="/wiki/Flux_(text-to-image_model)" title="Flux (text-to-image model)">Flux</a></li> <li><a href="/wiki/Ideogram_(text-to-image_model)" title="Ideogram (text-to-image model)">Ideogram</a></li> <li><a href="/wiki/Imagen_(text-to-image_model)" title="Imagen (text-to-image model)">Imagen</a></li> <li><a href="/wiki/Midjourney" title="Midjourney">Midjourney</a></li> <li><a href="/wiki/Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a></li></ul></li> <li><a href="/wiki/Text-to-video_model" title="Text-to-video model">Text-to-video models</a> <ul><li><a href="/wiki/Dream_Machine_(text-to-video_model)" title="Dream Machine (text-to-video model)">Dream Machine</a></li> <li><a href="/wiki/Runway_(company)#Gen-3_Alpha" title="Runway (company)">Gen-3 Alpha</a></li> <li><a href="/wiki/MiniMax_(company)#Hailuo_AI" title="MiniMax (company)">Hailuo AI</a></li> <li><a href="/wiki/Kling_(text-to-video_model)" class="mw-redirect" title="Kling (text-to-video model)">Kling</a></li> <li><a href="/wiki/Sora_(text-to-video_model)" title="Sora (text-to-video model)">Sora</a></li> <li><a href="/wiki/Google_DeepMind#Video_model" title="Google DeepMind">Veo</a></li></ul></li> <li><a href="/wiki/Music_and_artificial_intelligence" title="Music and artificial intelligence">Music generation</a> <ul><li><a href="/wiki/Suno_AI" title="Suno AI">Suno AI</a></li> <li><a href="/wiki/Udio" title="Udio">Udio</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Text</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Word2vec" title="Word2vec">Word2vec</a></li> <li><a href="/wiki/Seq2seq" title="Seq2seq">Seq2seq</a></li> <li><a href="/wiki/GloVe" title="GloVe">GloVe</a></li> <li><a href="/wiki/BERT_(language_model)" title="BERT (language model)">BERT</a></li> <li><a href="/wiki/T5_(language_model)" title="T5 (language model)">T5</a></li> <li><a href="/wiki/Llama_(language_model)" title="Llama (language model)">Llama</a></li> <li><a href="/wiki/Chinchilla_(language_model)" title="Chinchilla (language model)">Chinchilla AI</a></li> <li><a href="/wiki/PaLM" title="PaLM">PaLM</a></li> <li><a href="/wiki/Generative_pre-trained_transformer" title="Generative pre-trained transformer">GPT</a> <ul><li><a href="/wiki/GPT-1" title="GPT-1">1</a></li> <li><a href="/wiki/GPT-2" title="GPT-2">2</a></li> <li><a href="/wiki/GPT-3" title="GPT-3">3</a></li> <li><a href="/wiki/GPT-J" title="GPT-J">J</a></li> <li><a href="/wiki/ChatGPT" title="ChatGPT">ChatGPT</a></li> <li><a href="/wiki/GPT-4" title="GPT-4">4</a></li> <li><a href="/wiki/GPT-4o" title="GPT-4o">4o</a></li> <li><a href="/wiki/GPT-4.5" title="GPT-4.5">4.5</a></li> <li><a href="/wiki/OpenAI_o1" title="OpenAI o1">o1</a></li> <li><a href="/wiki/OpenAI_o3" title="OpenAI o3">o3</a></li></ul></li> <li><a href="/wiki/Claude_(language_model)" title="Claude (language model)">Claude</a></li> <li><a href="/wiki/Gemini_(language_model)" title="Gemini (language model)">Gemini</a> <ul><li><a href="/wiki/Gemini_(chatbot)" title="Gemini (chatbot)">chatbot</a></li></ul></li> <li><a href="/wiki/Grok_(chatbot)" title="Grok (chatbot)">Grok</a></li> <li><a href="/wiki/LaMDA" title="LaMDA">LaMDA</a></li> <li><a href="/wiki/BLOOM_(language_model)" title="BLOOM (language model)">BLOOM</a></li> <li><a href="/wiki/Project_Debater" title="Project Debater">Project Debater</a></li> <li><a href="/wiki/IBM_Watson" title="IBM Watson">IBM Watson</a></li> <li><a href="/wiki/IBM_Watsonx" title="IBM Watsonx">IBM Watsonx</a></li> <li><a href="/wiki/IBM_Granite" title="IBM Granite">Granite</a></li> <li><a href="/wiki/Huawei_PanGu" title="Huawei PanGu">PanGu-Σ</a></li> <li><a href="/wiki/DeepSeek_(chatbot)" title="DeepSeek (chatbot)">DeepSeek</a></li> <li><a href="/wiki/Qwen" title="Qwen">Qwen</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Decisional</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/AlphaGo" title="AlphaGo">AlphaGo</a></li> <li><a href="/wiki/AlphaZero" title="AlphaZero">AlphaZero</a></li> <li><a href="/wiki/OpenAI_Five" title="OpenAI Five">OpenAI Five</a></li> <li><a href="/wiki/Self-driving_car" title="Self-driving car">Self-driving car</a></li> <li><a href="/wiki/MuZero" title="MuZero">MuZero</a></li> <li><a href="/wiki/Action_selection" title="Action selection">Action selection</a> <ul><li><a href="/wiki/AutoGPT" title="AutoGPT">AutoGPT</a></li></ul></li> <li><a href="/wiki/Robot_control" title="Robot control">Robot control</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Alan_Turing" title="Alan Turing">Alan Turing</a></li> <li><a href="/wiki/Warren_Sturgis_McCulloch" title="Warren Sturgis McCulloch">Warren Sturgis McCulloch</a></li> <li><a href="/wiki/Walter_Pitts" title="Walter Pitts">Walter Pitts</a></li> <li><a href="/wiki/John_von_Neumann" title="John von Neumann">John von Neumann</a></li> <li><a href="/wiki/Claude_Shannon" title="Claude Shannon">Claude Shannon</a></li> <li><a href="/wiki/Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a></li> <li><a href="/wiki/John_McCarthy_(computer_scientist)" title="John McCarthy (computer scientist)">John McCarthy</a></li> <li><a href="/wiki/Nathaniel_Rochester_(computer_scientist)" title="Nathaniel Rochester (computer scientist)">Nathaniel Rochester</a></li> <li><a href="/wiki/Allen_Newell" title="Allen Newell">Allen Newell</a></li> <li><a href="/wiki/Cliff_Shaw" title="Cliff Shaw">Cliff Shaw</a></li> <li><a href="/wiki/Herbert_A._Simon" title="Herbert A. Simon">Herbert A. Simon</a></li> <li><a href="/wiki/Oliver_Selfridge" title="Oliver Selfridge">Oliver Selfridge</a></li> <li><a href="/wiki/Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a></li> <li><a href="/wiki/Bernard_Widrow" title="Bernard Widrow">Bernard Widrow</a></li> <li><a href="/wiki/Joseph_Weizenbaum" title="Joseph Weizenbaum">Joseph Weizenbaum</a></li> <li><a href="/wiki/Seymour_Papert" title="Seymour Papert">Seymour Papert</a></li> <li><a href="/wiki/Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a></li> <li><a href="/wiki/Paul_Werbos" title="Paul Werbos">Paul Werbos</a></li> <li><a href="/wiki/J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a></li> <li><a href="/wiki/Yann_LeCun" title="Yann LeCun">Yann LeCun</a></li> <li><a href="/wiki/Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li> <li><a href="/wiki/John_Hopfield" title="John Hopfield">John Hopfield</a></li> <li><a href="/wiki/Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li> <li><a href="/wiki/Lotfi_A._Zadeh" title="Lotfi A. Zadeh">Lotfi A. Zadeh</a></li> <li><a href="/wiki/Stephen_Grossberg" title="Stephen Grossberg">Stephen Grossberg</a></li> <li><a href="/wiki/Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Alex Graves</a></li> <li><a href="/wiki/Andrew_Ng" title="Andrew Ng">Andrew Ng</a></li> <li><a href="/wiki/Fei-Fei_Li" title="Fei-Fei Li">Fei-Fei Li</a></li> <li><a href="/wiki/Alex_Krizhevsky" title="Alex Krizhevsky">Alex Krizhevsky</a></li> <li><a href="/wiki/Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li> <li><a href="/wiki/Demis_Hassabis" title="Demis Hassabis">Demis Hassabis</a></li> <li><a href="/wiki/David_Silver_(computer_scientist)" title="David Silver (computer scientist)">David Silver</a></li> <li><a href="/wiki/Ian_Goodfellow" title="Ian Goodfellow">Ian Goodfellow</a></li> <li><a href="/wiki/Andrej_Karpathy" title="Andrej Karpathy">Andrej Karpathy</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Architectures</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a></li> <li><a href="/wiki/Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></li> <li><a href="/wiki/Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a> <ul><li><a href="/wiki/Vision_transformer" title="Vision transformer">Vision transformer (ViT)</a></li></ul></li> <li><a href="/wiki/Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network (RNN)</a></li> <li><a href="/wiki/Long_short-term_memory" title="Long short-term memory">Long short-term memory (LSTM)</a></li> <li><a href="/wiki/Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit (GRU)</a></li> <li><a href="/wiki/Echo_state_network" title="Echo state network">Echo state network</a></li> <li><a href="/wiki/Multilayer_perceptron" title="Multilayer perceptron">Multilayer perceptron (MLP)</a></li> <li><a href="/wiki/Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network (CNN)</a></li> <li><a href="/wiki/Residual_neural_network" title="Residual neural network">Residual neural network (RNN)</a></li> <li><a href="/wiki/Highway_network" title="Highway network">Highway network</a></li> <li><a href="/wiki/Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li> <li><a class="mw-selflink selflink">Autoencoder</a></li> <li><a href="/wiki/Variational_autoencoder" title="Variational autoencoder">Variational autoencoder (VAE)</a></li> <li><a href="/wiki/Generative_adversarial_network" title="Generative adversarial network">Generative adversarial network (GAN)</a></li> <li><a href="/wiki/Graph_neural_network" title="Graph neural network">Graph neural network (GNN)</a></li></ul> </div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><span class="noviewer" typeof="mw:File"><a href="/wiki/File:Symbol_portal_class.svg" class="mw-file-description" title="Portal"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/16px-Symbol_portal_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/23px-Symbol_portal_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/31px-Symbol_portal_class.svg.png 2x" data-file-width="180" data-file-height="185" /></a></span> Portals <ul><li><a href="/wiki/Portal:Technology" title="Portal:Technology">Technology</a></li></ul></li> <li><span class="noviewer" typeof="mw:File"><span title="Category"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/16px-Symbol_category_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/23px-Symbol_category_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/31px-Symbol_category_class.svg.png 2x" data-file-width="180" data-file-height="185" /></span></span> <a href="/wiki/Category:Artificial_intelligence" title="Category:Artificial intelligence">Category</a> <ul><li><a href="/wiki/Category:Artificial_neural_networks" title="Category:Artificial neural networks">Artificial neural networks</a></li> <li><a href="/wiki/Category:Machine_learning" title="Category:Machine learning">Machine learning</a></li></ul></li> <li><span class="noviewer" typeof="mw:File"><span title="List-Class article"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/d/db/Symbol_list_class.svg/16px-Symbol_list_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/d/db/Symbol_list_class.svg/23px-Symbol_list_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/d/db/Symbol_list_class.svg/31px-Symbol_list_class.svg.png 2x" data-file-width="180" data-file-height="185" /></span></span> List <ul><li><a href="/wiki/List_of_artificial_intelligence_companies" title="List of artificial intelligence companies">Companies</a></li> <li><a href="/wiki/List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235" /></div><div role="navigation" class="navbox" aria-labelledby="Noise_(physics_and_telecommunications)164" style="padding:3px"><table class="nowraplinks mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1239400231" /><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Noise" title="Template:Noise"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Noise" title="Template talk:Noise"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Noise" title="Special:EditPage/Template:Noise"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Noise_(physics_and_telecommunications)164" style="font-size:114%;margin:0 4em"><a href="/wiki/Noise_(spectral_phenomenon)" title="Noise (spectral phenomenon)">Noise</a> (physics and telecommunications)</div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%">General</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Acoustic_quieting" title="Acoustic quieting">Acoustic quieting</a></li> <li><a href="/wiki/Distortion" title="Distortion">Distortion</a></li> <li><a href="/wiki/Active_noise_control" title="Active noise control">Noise cancellation</a></li> <li><a href="/wiki/Noise_control" title="Noise control">Noise control</a></li> <li><a href="/wiki/Noise_measurement" title="Noise measurement">Noise measurement</a></li> <li><a href="/wiki/Noise_power" title="Noise power">Noise power</a></li> <li><a href="/wiki/Noise_reduction" title="Noise reduction">Noise reduction</a></li> <li><a href="/wiki/Noise_temperature" title="Noise temperature">Noise temperature</a></li> <li><a href="/wiki/Phase_distortion" title="Phase distortion">Phase distortion</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Noise in...</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Noise" title="Noise">Audio</a></li> <li><a href="/wiki/Architectural_acoustics" title="Architectural acoustics">Buildings</a></li> <li><a href="/wiki/Noise_(electronics)" title="Noise (electronics)">Electronics</a></li> <li><a href="/wiki/Noise_pollution" title="Noise pollution">Environment</a></li> <li><a href="/wiki/Noise_regulation" title="Noise regulation">Government regulation</a></li> <li><a href="/wiki/Health_effects_from_noise" title="Health effects from noise">Human health</a></li> <li><a href="/wiki/Image_noise" title="Image noise">Images</a></li> <li><a href="/wiki/Noise_(radio)" class="mw-redirect" title="Noise (radio)">Radio</a></li> <li><a href="/wiki/Soundproofing" title="Soundproofing">Rooms</a></li> <li><a href="/wiki/Noise_and_vibration_on_maritime_vessels" title="Noise and vibration on maritime vessels">Ships</a></li> <li><a href="/wiki/Sound_masking" title="Sound masking">Sound masking</a></li> <li><a href="/wiki/Noise_barrier" title="Noise barrier">Transportation</a></li> <li><a href="/wiki/Noise_(video)" title="Noise (video)">Video</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Class of noise</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Additive_white_Gaussian_noise" title="Additive white Gaussian noise">Additive white Gaussian noise</a> (AWGN)</li> <li><a href="/wiki/Atmospheric_noise" title="Atmospheric noise">Atmospheric noise</a></li> <li><a href="/wiki/Background_noise" title="Background noise">Background noise</a></li> <li><a href="/wiki/Brownian_noise" title="Brownian noise">Brownian noise</a></li> <li><a href="/wiki/Burst_noise" title="Burst noise">Burst noise</a></li> <li><a href="/wiki/Cosmic_noise" title="Cosmic noise">Cosmic noise</a></li> <li><a href="/wiki/Flicker_noise" title="Flicker noise">Flicker noise</a></li> <li><a href="/wiki/Gaussian_noise" title="Gaussian noise">Gaussian noise</a></li> <li><a href="/wiki/Grey_noise" title="Grey noise">Grey noise</a></li> <li><a href="/wiki/Infrasound" title="Infrasound">Infrasound</a></li> <li><a href="/wiki/Jitter" title="Jitter">Jitter</a></li> <li><a href="/wiki/Johnson%E2%80%93Nyquist_noise" title="Johnson–Nyquist noise">Johnson–Nyquist noise</a> (thermal noise)</li> <li><a href="/wiki/Pink_noise" title="Pink noise">Pink noise</a></li> <li><a href="/wiki/Quantization_error" class="mw-redirect" title="Quantization error">Quantization error</a> (or q. noise)</li> <li><a href="/wiki/Shot_noise" title="Shot noise">Shot noise</a></li> <li><a href="/wiki/White_noise" title="White noise">White noise</a></li> <li>Coherent noise <ul><li><a href="/wiki/Value_noise" title="Value noise">Value noise</a></li> <li><a href="/wiki/Gradient_noise" title="Gradient noise">Gradient noise</a></li> <li><a href="/wiki/Worley_noise" title="Worley noise">Worley noise</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Engineering <br />terms</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Channel_noise_level" title="Channel noise level">Channel noise level</a></li> <li><a href="/wiki/Circuit_noise_level" title="Circuit noise level">Circuit noise level</a></li> <li><a href="/wiki/Effective_input_noise_temperature" title="Effective input noise temperature">Effective input noise temperature</a></li> <li><a href="/wiki/Equivalent_noise_resistance" title="Equivalent noise resistance">Equivalent noise resistance</a></li> <li><a href="/wiki/Equivalent_pulse_code_modulation_noise" title="Equivalent pulse code modulation noise">Equivalent pulse code modulation noise</a></li> <li><a href="/wiki/Impulse_noise_(audio)" class="mw-redirect" title="Impulse noise (audio)">Impulse noise (audio)</a></li> <li><a href="/wiki/Noise_figure" title="Noise figure">Noise figure</a></li> <li><a href="/wiki/Noise_floor" title="Noise floor">Noise floor</a></li> <li><a href="/wiki/Noise_shaping" title="Noise shaping">Noise shaping</a></li> <li><a href="/wiki/Noise_spectral_density" title="Noise spectral density">Noise spectral density</a></li> <li><a href="/wiki/Noise,_vibration,_and_harshness" title="Noise, vibration, and harshness">Noise, vibration, and harshness</a> (NVH)</li> <li><a href="/wiki/Phase_noise" title="Phase noise">Phase noise</a></li> <li><a href="/wiki/Pseudorandom_noise" title="Pseudorandom noise">Pseudorandom noise</a></li> <li><a href="/wiki/Statistical_noise" class="mw-redirect" title="Statistical noise">Statistical noise</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Ratios</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Carrier-to-noise_ratio" title="Carrier-to-noise ratio">Carrier-to-noise ratio</a> (<i>C</i>/<i>N</i>)</li> <li><a href="/wiki/Carrier-to-receiver_noise_density" class="mw-redirect" title="Carrier-to-receiver noise density">Carrier-to-receiver noise density</a> (<i>C</i>/<i>kT</i>)</li> <li><i><a href="/wiki/DBrnC" class="mw-redirect" title="DBrnC">dBrnC</a></i></li> <li><i><a href="/wiki/Eb/N0" title="Eb/N0">E<sub>b</sub>/N<sub>0</sub></a></i> (energy per bit to noise density)</li> <li><i><a href="/wiki/Eb/N0#Relation_to_Es.2FN0" title="Eb/N0">E<sub>s</sub>/N<sub>0</sub></a></i> (energy per symbol to noise density)</li> <li><a href="/wiki/Modulation_error_ratio" title="Modulation error ratio">Modulation error ratio</a> (<i>MER</i>)</li> <li><a href="/wiki/SINAD" title="SINAD">Signal, noise and distortion</a> (<i>SINAD</i>)</li> <li><a href="/wiki/Signal-to-interference_ratio" title="Signal-to-interference ratio">Signal-to-interference ratio</a> (<i>S</i>/<i>I</i>)</li> <li><a href="/wiki/Signal-to-noise_ratio" title="Signal-to-noise ratio">Signal-to-noise ratio</a> (<i>S</i>/<i>N</i>, <i>SNR</i>)</li> <li><a href="/wiki/Signal-to-noise_ratio_(imaging)" title="Signal-to-noise ratio (imaging)">Signal-to-noise ratio (imaging)</a></li> <li><a href="/wiki/Signal-to-interference-plus-noise_ratio" title="Signal-to-interference-plus-noise ratio">Signal-to-interference-plus-noise ratio</a> (<i>SINR</i>)</li> <li><a href="/wiki/Signal-to-quantization-noise_ratio" title="Signal-to-quantization-noise ratio">Signal-to-quantization-noise ratio</a> (<i>SQNR</i>)</li> <li><a href="/wiki/Contrast-to-noise_ratio" title="Contrast-to-noise ratio">Contrast-to-noise ratio</a> (<i>CNR</i>)</li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Related topics</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/List_of_noise_topics" title="List of noise topics">List of noise topics</a></li> <li><a href="/wiki/Acoustics" title="Acoustics">Acoustics</a></li> <li><a href="/wiki/Colors_of_noise" title="Colors of noise">Colors of noise</a></li> <li><a href="/wiki/Interference_(communication)" title="Interference (communication)">Interference (communication)</a></li> <li><a href="/wiki/Noise_generator" title="Noise generator">Noise generator</a></li> <li><a href="/wiki/Spectrum_analyzer" title="Spectrum analyzer">Spectrum analyzer</a></li> <li><a href="/wiki/Thermal_radiation" title="Thermal radiation">Thermal radiation</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Denoise <br />methods</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">General</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Low-pass_filter" title="Low-pass filter">Low-pass filter</a></li> <li><a href="/wiki/Median_filter" title="Median filter">Median filter</a></li> <li><a href="/wiki/Total_variation_denoising" title="Total variation denoising">Total variation denoising</a></li> <li><a href="/wiki/Wavelet#Wavelet_denoising" title="Wavelet">Wavelet denoising</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">2D (Image)</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Gaussian_blur" title="Gaussian blur">Gaussian blur</a></li> <li><a href="/wiki/Anisotropic_diffusion" title="Anisotropic diffusion">Anisotropic diffusion</a></li> <li><a href="/wiki/Bilateral_filter" title="Bilateral filter">Bilateral filter</a></li> <li><a href="/wiki/Non-local_means" title="Non-local means">Non-local means</a></li> <li><a href="/wiki/Block-matching_and_3D_filtering" title="Block-matching and 3D filtering">Block-matching and 3D filtering</a> (BM3D)</li> <li><a href="/wiki/Shrinkage_Fields_(image_restoration)" title="Shrinkage Fields (image restoration)">Shrinkage Fields</a></li> <li><a class="mw-selflink-fragment" href="#Denoising_autoencoder_(DAE)">Denoising autoencoder</a> (DAE)</li> <li><a href="/wiki/Deep_Image_Prior" class="mw-redirect" title="Deep Image Prior">Deep Image Prior</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?useformat=desktop&type=1x1&usesul3=0" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Autoencoder&oldid=1275135656">https://en.wikipedia.org/w/index.php?title=Autoencoder&oldid=1275135656</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Neural_network_architectures" title="Category:Neural network architectures">Neural network architectures</a></li><li><a href="/wiki/Category:Unsupervised_learning" title="Category:Unsupervised learning">Unsupervised learning</a></li><li><a href="/wiki/Category:Dimension_reduction" title="Category:Dimension reduction">Dimension reduction</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_matches_Wikidata" title="Category:Short description matches Wikidata">Short description matches Wikidata</a></li><li><a href="/wiki/Category:Use_dmy_dates_from_March_2020" title="Category:Use dmy dates from March 2020">Use dmy dates from March 2020</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 11 February 2025, at 07:35<span class="anonymous-show"> (UTC)</span>.</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Autoencoder&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><picture><source media="(min-width: 500px)" srcset="/static/images/footer/wikimedia-button.svg" width="84" height="29"><img src="/static/images/footer/wikimedia.svg" width="25" height="25" alt="Wikimedia Foundation" lang="en" loading="lazy"></picture></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><picture><source media="(min-width: 500px)" srcset="/w/resources/assets/poweredby_mediawiki.svg" width="88" height="31"><img src="/w/resources/assets/mediawiki_compact.svg" alt="Powered by MediaWiki" lang="en" width="25" height="25" loading="lazy"></picture></a></li> </ul> </footer> </div> </div> </div> <div class="vector-header-container vector-sticky-header-container"> <div id="vector-sticky-header" class="vector-sticky-header"> <div class="vector-sticky-header-start"> <div class="vector-sticky-header-icon-start vector-button-flush-left vector-button-flush-right" aria-hidden="true"> <button class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-sticky-header-search-toggle" tabindex="-1" data-event-name="ui.vector-sticky-search-form.icon"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </button> </div> <div role="search" class="vector-search-box-vue vector-search-box-show-thumbnail vector-search-box"> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail"> <form action="/w/index.php" id="vector-sticky-search-form" class="cdx-search-input cdx-search-input--has-end-button"> <div class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia"> <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <div class="vector-sticky-header-context-bar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-sticky-header-toc" class="vector-dropdown mw-portlet mw-portlet-sticky-header-toc vector-sticky-header-toc vector-button-flush-left" > <input type="checkbox" id="vector-sticky-header-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-sticky-header-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-sticky-header-toc-label" for="vector-sticky-header-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-sticky-header-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div class="vector-sticky-header-context-bar-primary" aria-hidden="true" ><span class="mw-page-title-main">Autoencoder</span></div> </div> </div> <div class="vector-sticky-header-end" aria-hidden="true"> <div class="vector-sticky-header-icons"> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-talk-sticky-header" tabindex="-1" data-event-name="talk-sticky-header"><span class="vector-icon mw-ui-icon-speechBubbles mw-ui-icon-wikimedia-speechBubbles"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-subject-sticky-header" tabindex="-1" data-event-name="subject-sticky-header"><span class="vector-icon mw-ui-icon-article mw-ui-icon-wikimedia-article"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-history-sticky-header" tabindex="-1" data-event-name="history-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-history mw-ui-icon-wikimedia-wikimedia-history"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only mw-watchlink" id="ca-watchstar-sticky-header" tabindex="-1" data-event-name="watch-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-star mw-ui-icon-wikimedia-wikimedia-star"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-edit-sticky-header" tabindex="-1" data-event-name="wikitext-edit-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-wikiText mw-ui-icon-wikimedia-wikimedia-wikiText"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-ve-edit-sticky-header" tabindex="-1" data-event-name="ve-edit-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-edit mw-ui-icon-wikimedia-wikimedia-edit"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-viewsource-sticky-header" tabindex="-1" data-event-name="ve-edit-protected-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-editLock mw-ui-icon-wikimedia-wikimedia-editLock"></span> <span></span> </a> </div> <div class="vector-sticky-header-buttons"> <button class="cdx-button cdx-button--weight-quiet mw-interlanguage-selector" id="p-lang-btn-sticky-header" tabindex="-1" data-event-name="ui.dropdown-p-lang-btn-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>20 languages</span> </button> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive" id="ca-addsection-sticky-header" tabindex="-1" data-event-name="addsection-sticky-header"><span class="vector-icon mw-ui-icon-speechBubbleAdd-progressive mw-ui-icon-wikimedia-speechBubbleAdd-progressive"></span> <span>Add topic</span> </a> </div> <div class="vector-sticky-header-icon-end"> <div class="vector-user-links"> </div> </div> </div> </div> </div> <div class="mw-portlet mw-portlet-dock-bottom emptyPortlet" id="p-dock-bottom"> <ul> </ul> </div> <script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.eqiad.main-8669bc5c8-57zq6","wgBackendResponseTime":291,"wgPageParseReport":{"limitreport":{"cputime":"1.057","walltime":"1.566","ppvisitednodes":{"value":5051,"limit":1000000},"postexpandincludesize":{"value":227821,"limit":2097152},"templateargumentsize":{"value":3541,"limit":2097152},"expansiondepth":{"value":12,"limit":100},"expensivefunctioncount":{"value":5,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":261091,"limit":5000000},"entityaccesscount":{"value":0,"limit":400},"timingprofile":["100.00% 895.030 1 -total"," 45.03% 403.057 1 Template:Reflist"," 20.24% 181.196 29 Template:Cite_journal"," 18.59% 166.400 15 Template:Cite_book"," 13.05% 116.766 1 Template:Machine_learning"," 12.18% 109.018 1 Template:Sidebar_with_collapsible_lists"," 11.00% 98.480 1 Template:Short_description"," 8.26% 73.922 12 Template:Cite_arXiv"," 7.65% 68.429 4 Template:Navbox"," 7.00% 62.660 1 Template:Artificial_intelligence_navbox"]},"scribunto":{"limitreport-timeusage":{"value":"0.572","limit":"10.000"},"limitreport-memusage":{"value":6264857,"limit":52428800}},"cachereport":{"origin":"mw-web.eqiad.main-8669bc5c8-k5m99","timestamp":"20250318160450","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Autoencoder","url":"https:\/\/en.wikipedia.org\/wiki\/Autoencoder","sameAs":"http:\/\/www.wikidata.org\/entity\/Q786435","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q786435","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2006-09-04T09:11:50Z","dateModified":"2025-02-11T07:35:16Z","image":"https:\/\/upload.wikimedia.org\/wikipedia\/commons\/3\/37\/Autoencoder_schema.png","headline":"neural network that learns efficient data encoding in an unsupervised manner"}</script> </body> </html>