Automatic differentiation

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Automatic differentiation - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"1b307676-44e6-463b-a615-21fc65ad46bc","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Automatic_differentiation","wgTitle":"Automatic differentiation","wgCurRevisionId":1283271943,"wgRevisionId":1283271943,"wgArticleId":734787,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Articles with imported Creative Commons Attribution-ShareAlike 4.0 text","CS1 maint: multiple names: authors list","Articles with short description","Short description is different from Wikidata","Webarchive template wayback links","Articles with example pseudocode","Articles with example Python (programming language) code","Articles with example C++ code","Differential calculus","Computer algebra"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Automatic_differentiation","wgRelevantArticleId":734787,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":40000,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q787371","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGELevelingUpEnabledForUser":false}; RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.cite.styles":"ready","ext.math.styles":"ready","ext.pygments":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","ext.pygments.view","mediawiki.page.media","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.quicksurveys.init","ext.growthExperiments.SuggestedEditSession"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.pygments%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.22"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Automatic differentiation - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Automatic_differentiation"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Automatic_differentiation&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Automatic_differentiation"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="auth.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Automatic_differentiation rootpage-Automatic_differentiation skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" title="Main menu" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Main menu </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z">Main page</a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia">Contents</a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events">Current events</a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x">Random article</a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works">About Wikipedia</a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia">Contact us</a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia">Help</a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia">Learn to edit</a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors">Community portal</a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r">Recent changes</a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia">Upload file</a></li><li id="n-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages">Special pages</a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"> Search </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Appearance </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=en.wikipedia.org&uselang=en" class="">Donate</a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Automatic+differentiation" title="You are encouraged to create an account and log in; however, it is not mandatory" class="">Create account</a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Automatic+differentiation" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class="">Log in</a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Personal tools </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=en.wikipedia.org&uselang=en">Donate</a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Automatic+differentiation" title="You are encouraged to create an account and log in; however, it is not mandatory"> Create account</a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Automatic+differentiation" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"> Log in</a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing">learn more</a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y">Contributions</a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n">Talk</a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-Difference_from_other_differentiation_methods" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Difference_from_other_differentiation_methods"> <div class="vector-toc-text"> 1 Difference from other differentiation methods </div> </a> <ul id="toc-Difference_from_other_differentiation_methods-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Applications" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Applications"> <div class="vector-toc-text"> 2 Applications </div> </a> <ul id="toc-Applications-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Forward_and_reverse_accumulation" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Forward_and_reverse_accumulation"> <div class="vector-toc-text"> 3 Forward and reverse accumulation </div> </a> <button aria-controls="toc-Forward_and_reverse_accumulation-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Forward and reverse accumulation subsection </button> <ul id="toc-Forward_and_reverse_accumulation-sublist" class="vector-toc-list"> <li id="toc-Chain_rule_of_partial_derivatives_of_composite_functions" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Chain_rule_of_partial_derivatives_of_composite_functions"> <div class="vector-toc-text"> 3.1 Chain rule of partial derivatives of composite functions </div> </a> <ul id="toc-Chain_rule_of_partial_derivatives_of_composite_functions-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Two_types_of_automatic_differentiation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Two_types_of_automatic_differentiation"> <div class="vector-toc-text"> 3.2 Two types of automatic differentiation </div> </a> <ul id="toc-Two_types_of_automatic_differentiation-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Forward_accumulation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Forward_accumulation"> <div class="vector-toc-text"> 3.3 Forward accumulation </div> </a> <ul id="toc-Forward_accumulation-sublist" class="vector-toc-list"> <li id="toc-Implementation" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Implementation"> <div class="vector-toc-text"> 3.3.1 Implementation </div> </a> <ul id="toc-Implementation-sublist" class="vector-toc-list"> <li id="toc-Pseudocode" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#Pseudocode"> <div class="vector-toc-text"> 3.3.1.1 Pseudocode </div> </a> <ul id="toc-Pseudocode-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-C++" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#C++"> <div class="vector-toc-text"> 3.3.1.2 C++ </div> </a> <ul id="toc-C++-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> </ul> </li> <li id="toc-Reverse_accumulation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Reverse_accumulation"> <div class="vector-toc-text"> 3.4 Reverse accumulation </div> </a> <ul id="toc-Reverse_accumulation-sublist" class="vector-toc-list"> <li id="toc-Implementation_2" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Implementation_2"> <div class="vector-toc-text"> 3.4.1 Implementation </div> </a> <ul id="toc-Implementation_2-sublist" class="vector-toc-list"> <li id="toc-Pseudo_code" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#Pseudo_code"> <div class="vector-toc-text"> 3.4.1.1 Pseudo code </div> </a> <ul id="toc-Pseudo_code-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-C++_2" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#C++_2"> <div class="vector-toc-text"> 3.4.1.2 C++ </div> </a> <ul id="toc-C++_2-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> </ul> </li> <li id="toc-Beyond_forward_and_reverse_accumulation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Beyond_forward_and_reverse_accumulation"> <div class="vector-toc-text"> 3.5 Beyond forward and reverse accumulation </div> </a> <ul id="toc-Beyond_forward_and_reverse_accumulation-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Automatic_differentiation_using_dual_numbers" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Automatic_differentiation_using_dual_numbers"> <div class="vector-toc-text"> 4 Automatic differentiation using dual numbers </div> </a> <button aria-controls="toc-Automatic_differentiation_using_dual_numbers-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Automatic differentiation using dual numbers subsection </button> <ul id="toc-Automatic_differentiation_using_dual_numbers-sublist" class="vector-toc-list"> <li id="toc-Implementation_3" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Implementation_3"> <div class="vector-toc-text"> 4.1 Implementation </div> </a> <ul id="toc-Implementation_3-sublist" class="vector-toc-list"> <li id="toc-Pseudo_code_2" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Pseudo_code_2"> <div class="vector-toc-text"> 4.1.1 Pseudo code </div> </a> <ul id="toc-Pseudo_code_2-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-C++_3" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#C++_3"> <div class="vector-toc-text"> 4.1.2 C++ </div> </a> <ul id="toc-C++_3-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Vector_arguments_and_functions" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Vector_arguments_and_functions"> <div class="vector-toc-text"> 4.2 Vector arguments and functions </div> </a> <ul id="toc-Vector_arguments_and_functions-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-High_order_and_many_variables" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#High_order_and_many_variables"> <div class="vector-toc-text"> 4.3 High order and many variables </div> </a> <ul id="toc-High_order_and_many_variables-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Implementation_4" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Implementation_4"> <div class="vector-toc-text"> 5 Implementation </div> </a> <button aria-controls="toc-Implementation_4-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> Toggle Implementation subsection </button> <ul id="toc-Implementation_4-sublist" class="vector-toc-list"> <li id="toc-Source_code_transformation_(SCT)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Source_code_transformation_(SCT)"> <div class="vector-toc-text"> 5.1 Source code transformation (SCT) </div> </a> <ul id="toc-Source_code_transformation_(SCT)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Operator_overloading_(OO)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Operator_overloading_(OO)"> <div class="vector-toc-text"> 5.2 Operator overloading (OO) </div> </a> <ul id="toc-Operator_overloading_(OO)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Operator_overloading_and_source_code_transformation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Operator_overloading_and_source_code_transformation"> <div class="vector-toc-text"> 5.3 Operator overloading and source code transformation </div> </a> <ul id="toc-Operator_overloading_and_source_code_transformation-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> 6 See also </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Notes" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Notes"> <div class="vector-toc-text"> 7 Notes </div> </a> <ul id="toc-Notes-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> 8 References </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Further_reading" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Further_reading"> <div class="vector-toc-text"> 9 Further reading </div> </a> <ul id="toc-Further_reading-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-External_links" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#External_links"> <div class="vector-toc-text"> 10 External links </div> </a> <ul id="toc-External_links-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" title="Table of Contents" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Toggle the table of contents </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading">Automatic differentiation</h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 11 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-11" aria-hidden="true" > 11 languages </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D8%A7%D9%84%D8%A7%D8%B4%D8%AA%D9%82%D8%A7%D9%82_%D8%A7%D9%84%D8%A2%D9%84%D9%8A" title="الاشتقاق الآلي – Arabic" lang="ar" hreflang="ar" data-title="الاشتقاق الآلي" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target">العربية</a></li><li class="interlanguage-link interwiki-ca mw-list-item"><a href="https://ca.wikipedia.org/wiki/Derivaci%C3%B3_autom%C3%A0tica" title="Derivació automàtica – Catalan" lang="ca" hreflang="ca" data-title="Derivació automàtica" data-language-autonym="Català" data-language-local-name="Catalan" class="interlanguage-link-target">Català</a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Automatisches_Differenzieren" title="Automatisches Differenzieren – German" lang="de" hreflang="de" data-title="Automatisches Differenzieren" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target">Deutsch</a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Diferenciaci%C3%B3n_autom%C3%A1tica" title="Diferenciación automática – Spanish" lang="es" hreflang="es" data-title="Diferenciación automática" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target">Español</a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/D%C3%A9rivation_automatique" title="Dérivation automatique – French" lang="fr" hreflang="fr" data-title="Dérivation automatique" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target">Français</a></li><li class="interlanguage-link interwiki-it mw-list-item"><a href="https://it.wikipedia.org/wiki/Differenziazione_automatica" title="Differenziazione automatica – Italian" lang="it" hreflang="it" data-title="Differenziazione automatica" data-language-autonym="Italiano" data-language-local-name="Italian" class="interlanguage-link-target">Italiano</a></li><li class="interlanguage-link interwiki-ja mw-list-item"><a href="https://ja.wikipedia.org/wiki/%E8%87%AA%E5%8B%95%E5%BE%AE%E5%88%86" title="自動微分 – Japanese" lang="ja" hreflang="ja" data-title="自動微分" data-language-autonym="日本語" data-language-local-name="Japanese" class="interlanguage-link-target">日本語</a></li><li class="interlanguage-link interwiki-pt mw-list-item"><a href="https://pt.wikipedia.org/wiki/Diferencia%C3%A7%C3%A3o_autom%C3%A1tica" title="Diferenciação automática – Portuguese" lang="pt" hreflang="pt" data-title="Diferenciação automática" data-language-autonym="Português" data-language-local-name="Portuguese" class="interlanguage-link-target">Português</a></li><li class="interlanguage-link interwiki-sr mw-list-item"><a href="https://sr.wikipedia.org/wiki/Automatska_diferencijacija" title="Automatska diferencijacija – Serbian" lang="sr" hreflang="sr" data-title="Automatska diferencijacija" data-language-autonym="Српски / srpski" data-language-local-name="Serbian" class="interlanguage-link-target">Српски / srpski</a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%90%D0%B2%D1%82%D0%BE%D0%BC%D0%B0%D1%82%D0%B8%D1%87%D0%BD%D0%B5_%D0%B4%D0%B8%D1%84%D0%B5%D1%80%D0%B5%D0%BD%D1%86%D1%96%D1%8E%D0%B2%D0%B0%D0%BD%D0%BD%D1%8F" title="Автоматичне диференціювання – Ukrainian" lang="uk" hreflang="uk" data-title="Автоматичне диференціювання" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target">Українська</a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E8%87%AA%E5%8B%95%E5%BE%AE%E5%88%86" title="自動微分 – Chinese" lang="zh" hreflang="zh" data-title="自動微分" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target">中文</a></li> </ul> <div class="after-portlet after-portlet-lang"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q787371#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Automatic_differentiation" title="View the content page [c]" accesskey="c">Article</a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Automatic_differentiation" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t">Talk</a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" >English </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Automatic_differentiation">Read</a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&action=edit" title="Edit this page [e]" accesskey="e">Edit</a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&action=history" title="Past revisions of this page [h]" accesskey="h">View history</a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" >Tools </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Automatic_differentiation">Read</a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&action=edit" title="Edit this page [e]" accesskey="e">Edit</a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&action=history">View history</a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Automatic_differentiation" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j">What links here</a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Automatic_differentiation" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li><li id="t-upload" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u">Upload file</a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&oldid=1283271943" title="Permanent link to this revision of this page">Permanent link</a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&action=info" title="More information about this page">Page information</a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Automatic_differentiation&id=1283271943&wpFormIdentifier=titleform" title="Information on how to cite this page">Cite this page</a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAutomatic_differentiation">Get shortened URL</a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAutomatic_differentiation">Download QR code</a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Automatic_differentiation&action=show-download-screen" title="Download this page as a PDF file">Download as PDF</a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Automatic_differentiation&printable=yes" title="Printable version of this page [p]" accesskey="p">Printable version</a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q787371" title="Structured data on this page hosted by Wikidata [g]" accesskey="g">Wikidata item</a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Numerical calculations carrying along derivatives</div> In <a href="/wiki/Mathematics" title="Mathematics">mathematics</a> and <a href="/wiki/Computer_algebra" title="Computer algebra">computer algebra</a>, automatic differentiation (auto-differentiation, autodiff, or AD), also called algorithmic differentiation, computational differentiation, and differentiation arithmetic<a href="#cite_note-1">[1]</a><a href="#cite_note-baydin2018automatic-2">[2]</a><a href="#cite_note-Dawood.Megahed.2023-3">[3]</a><a href="#cite_note-Dawood.Megahed.2019-4">[4]</a> is a set of techniques to evaluate the <a href="/wiki/Partial_derivative" title="Partial derivative">partial derivative</a> of a function specified by a computer program. Automatic differentiation is a subtle and central tool to automatize the simultaneous computation of the numerical values of arbitrarily complex functions and their derivatives with no need for the symbolic representation of the derivative, only the function rule or an algorithm thereof is required.<a href="#cite_note-Dawood.Megahed.2023-3">[3]</a><a href="#cite_note-Dawood.Megahed.2019-4">[4]</a> Auto-differentiation is thus neither numeric nor symbolic, nor is it a combination of both. It is also preferable to ordinary numerical methods: In contrast to the more traditional numerical methods based on finite differences, auto-differentiation is 'in theory' exact, and in comparison to symbolic algorithms, it is computationally inexpensive.<a href="#cite_note-Dawood-attribution-5">[5]</a><a href="#cite_note-Dawood.Megahed.2023-3">[3]</a><a href="#cite_note-Dawood.Dawood.2022-6">[6]</a> Automatic differentiation exploits the fact that every computer calculation, no matter how complicated, executes a sequence of elementary arithmetic operations (addition, subtraction, multiplication, division, etc.) and elementary functions (<a href="/wiki/Exponential_function" title="Exponential function">exp</a>, <a href="/wiki/Natural_logarithm" title="Natural logarithm">log</a>, <a href="/wiki/Sine" class="mw-redirect" title="Sine">sin</a>, <a href="/wiki/Cosine" class="mw-redirect" title="Cosine">cos</a>, etc.). By applying the <a href="/wiki/Chain_rule" title="Chain rule">chain rule</a> repeatedly to these operations, partial derivatives of arbitrary order can be computed automatically, accurately to working precision, and using at most a small constant factor of more arithmetic operations than the original program. <meta property="mw:PageProp/toc" /> <div class="mw-heading mw-heading2"><h2 id="Difference_from_other_differentiation_methods">Difference from other differentiation methods</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=1" title="Edit section: Difference from other differentiation methods">edit</a>]</div> <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:AutomaticDifferentiationNutshell.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/3/3c/AutomaticDifferentiationNutshell.png/330px-AutomaticDifferentiationNutshell.png" decoding="async" width="300" height="121" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/3c/AutomaticDifferentiationNutshell.png/500px-AutomaticDifferentiationNutshell.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3c/AutomaticDifferentiationNutshell.png/960px-AutomaticDifferentiationNutshell.png 2x" data-file-width="2244" data-file-height="908" /></a><figcaption>Figure 1: How automatic differentiation relates to symbolic differentiation</figcaption></figure> Automatic differentiation is distinct from <a href="/wiki/Symbolic_differentiation" class="mw-redirect" title="Symbolic differentiation">symbolic differentiation</a> and <a href="/wiki/Numerical_differentiation" title="Numerical differentiation">numerical differentiation</a>. Symbolic differentiation faces the difficulty of converting a computer program into a single <a href="/wiki/Mathematical_expression" class="mw-redirect" title="Mathematical expression">mathematical expression</a> and can lead to inefficient code. Numerical differentiation (the method of finite differences) can introduce <a href="/wiki/Round-off_error" title="Round-off error">round-off errors</a> in the <a href="/wiki/Discretization" title="Discretization">discretization</a> process and cancellation. Both of these classical methods have problems with calculating higher derivatives, where complexity and errors increase. Finally, both of these classical methods are slow at computing partial derivatives of a function with respect to many inputs, as is needed for <a href="/wiki/Gradient_descent" title="Gradient descent">gradient</a>-based <a href="/wiki/Optimization_(mathematics)" class="mw-redirect" title="Optimization (mathematics)">optimization</a> algorithms. Automatic differentiation solves all of these problems. <div class="mw-heading mw-heading2"><h2 id="Applications">Applications</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=2" title="Edit section: Applications">edit</a>]</div> Currently, for its efficiency and accuracy in computing first and higher order <a href="/wiki/Derivative" title="Derivative">derivatives</a>, auto-differentiation is a celebrated technique with diverse applications in <a href="/wiki/Scientific_computing" class="mw-redirect" title="Scientific computing">scientific computing</a> and <a href="/wiki/Mathematics" title="Mathematics">mathematics</a>. It should therefore come as no surprise that there are numerous computational implementations of auto-differentiation. Among these, one mentions <a href="/wiki/INTLAB" title="INTLAB">INTLAB</a>, <a href="/wiki/Sollya" class="mw-redirect" title="Sollya">Sollya</a>, and <a href="/w/index.php?title=InCLosure&action=edit&redlink=1" class="new" title="InCLosure (page does not exist)">InCLosure</a>.<a href="#cite_note-Rump.1999-7">[7]</a><a href="#cite_note-Chevillard.Joldes.Lauter.2010-8">[8]</a><a href="#cite_note-Dawood.Inc4.2022-9">[9]</a> In practice, there are two types (modes) of algorithmic differentiation: a forward-type and a reversed-type.<a href="#cite_note-Dawood.Megahed.2023-3">[3]</a><a href="#cite_note-Dawood.Megahed.2019-4">[4]</a> Presently, the two types are highly correlated and complementary and both have a wide variety of applications in, e.g., non-linear <a href="/wiki/Optimization" class="mw-redirect" title="Optimization">optimization</a>, <a href="/wiki/Sensitivity_analysis" title="Sensitivity analysis">sensitivity analysis</a>, <a href="/wiki/Robotics" title="Robotics">robotics</a>, <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a>, <a href="/wiki/Computer_graphics" title="Computer graphics">computer graphics</a>, and <a href="/wiki/Computer_vision" title="Computer vision">computer vision</a>.<a href="#cite_note-Dawood-attribution-5">[5]</a><a href="#cite_note-Fries.2019-10">[10]</a><a href="#cite_note-Dawood.Megahed.2023-3">[3]</a><a href="#cite_note-Dawood.Megahed.2019-4">[4]</a><a href="#cite_note-Dawood.Dawood.2020-11">[11]</a><a href="#cite_note-Dawood.2014-12">[12]</a> Automatic differentiation is particularly important in the field of <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a>. For example, it allows one to implement <a href="/wiki/Backpropagation" title="Backpropagation">backpropagation</a> in a <a href="/wiki/Neural_network_(machine_learning)" title="Neural network (machine learning)">neural network</a> without a manually-computed derivative. <div class="mw-heading mw-heading2"><h2 id="Forward_and_reverse_accumulation">Forward and reverse accumulation</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=3" title="Edit section: Forward and reverse accumulation">edit</a>]</div> <div class="mw-heading mw-heading3"><h3 id="Chain_rule_of_partial_derivatives_of_composite_functions">Chain rule of partial derivatives of composite functions</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=4" title="Edit section: Chain rule of partial derivatives of composite functions">edit</a>]</div> Fundamental to automatic differentiation is the decomposition of differentials provided by the <a href="/wiki/Chain_rule" title="Chain rule">chain rule</a> of <a href="/wiki/Partial_derivative" title="Partial derivative">partial derivatives</a> of <a href="/wiki/Function_composition" title="Function composition">composite functions</a>. For the simple composition <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}y&=f(g(h(x)))=f(g(h(w_{0})))=f(g(w_{1}))=f(w_{2})=w_{3}\\w_{0}&=x\\w_{1}&=h(w_{0})\\w_{2}&=g(w_{1})\\w_{3}&=f(w_{2})=y\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi>y</mi> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <mi>g</mi> <mo stretchy="false">(</mo> <mi>h</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <mi>g</mi> <mo stretchy="false">(</mo> <mi>h</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <mi>g</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">)</mo> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>x</mi> </mtd> </mtr> <mtr> <mtd> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>h</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>g</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mi>y</mi> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}y&=f(g(h(x)))=f(g(h(w_{0})))=f(g(w_{1}))=f(w_{2})=w_{3}\\w_{0}&=x\\w_{1}&=h(w_{0})\\w_{2}&=g(w_{1})\\w_{3}&=f(w_{2})=y\end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/616dc4060db7c22e60c53ee80fba10d7d7e2a9b3" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -7.171ex; width:58.589ex; height:15.509ex;" alt="{\displaystyle {\begin{aligned}y&=f(g(h(x)))=f(g(h(w_{0})))=f(g(w_{1}))=f(w_{2})=w_{3}\\w_{0}&=x\\w_{1}&=h(w_{0})\\w_{2}&=g(w_{1})\\w_{3}&=f(w_{2})=y\end{aligned}}}" /> the chain rule gives <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial y}{\partial x}}={\frac {\partial y}{\partial w_{2}}}{\frac {\partial w_{2}}{\partial w_{1}}}{\frac {\partial w_{1}}{\partial x}}={\frac {\partial f(w_{2})}{\partial w_{2}}}{\frac {\partial g(w_{1})}{\partial w_{1}}}{\frac {\partial h(w_{0})}{\partial x}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>f</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>g</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>h</mi> <mo stretchy="false">(</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial y}{\partial x}}={\frac {\partial y}{\partial w_{2}}}{\frac {\partial w_{2}}{\partial w_{1}}}{\frac {\partial w_{1}}{\partial x}}={\frac {\partial f(w_{2})}{\partial w_{2}}}{\frac {\partial g(w_{1})}{\partial w_{1}}}{\frac {\partial h(w_{0})}{\partial x}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/12416b908bbda13ccc4bb34450d9645f418f4792" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:48.078ex; height:6.176ex;" alt="{\displaystyle {\frac {\partial y}{\partial x}}={\frac {\partial y}{\partial w_{2}}}{\frac {\partial w_{2}}{\partial w_{1}}}{\frac {\partial w_{1}}{\partial x}}={\frac {\partial f(w_{2})}{\partial w_{2}}}{\frac {\partial g(w_{1})}{\partial w_{1}}}{\frac {\partial h(w_{0})}{\partial x}}}" /> <div class="mw-heading mw-heading3"><h3 id="Two_types_of_automatic_differentiation">Two types of automatic differentiation</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=5" title="Edit section: Two types of automatic differentiation">edit</a>]</div> Usually, two distinct modes of automatic differentiation are presented. <ul><li>forward accumulation (also called bottom-up, forward mode, or tangent mode)</li> <li>reverse accumulation (also called top-down, reverse mode, or adjoint mode)</li></ul> Forward accumulation specifies that one traverses the chain rule from inside to outside (that is, first compute <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial w_{1}/\partial x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \partial w_{1}/\partial x}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0d1c5296dac1de3f1307db636a6565448d653aa1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.847ex; height:2.843ex;" alt="{\displaystyle \partial w_{1}/\partial x}" /> and then <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial w_{2}/\partial w_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \partial w_{2}/\partial w_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/723364d6f3e44454623a65b06ad127bcbf632b54" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.235ex; height:2.843ex;" alt="{\displaystyle \partial w_{2}/\partial w_{1}}" /> and lastly <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial y/\partial w_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∂</mi> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \partial y/\partial w_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0954e0c2e2c2b8c4810bf7769aa7065f1d190b9f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.672ex; height:2.843ex;" alt="{\displaystyle \partial y/\partial w_{2}}" />), while reverse accumulation traverses from outside to inside (first compute <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial y/\partial w_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∂</mi> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \partial y/\partial w_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0954e0c2e2c2b8c4810bf7769aa7065f1d190b9f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.672ex; height:2.843ex;" alt="{\displaystyle \partial y/\partial w_{2}}" /> and then <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial w_{2}/\partial w_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \partial w_{2}/\partial w_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/723364d6f3e44454623a65b06ad127bcbf632b54" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.235ex; height:2.843ex;" alt="{\displaystyle \partial w_{2}/\partial w_{1}}" /> and lastly <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial w_{1}/\partial x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \partial w_{1}/\partial x}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0d1c5296dac1de3f1307db636a6565448d653aa1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.847ex; height:2.843ex;" alt="{\displaystyle \partial w_{1}/\partial x}" />). More succinctly, <ul><li>Forward accumulation computes the recursive relation: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial w_{i}}{\partial x}}={\frac {\partial w_{i}}{\partial w_{i-1}}}{\frac {\partial w_{i-1}}{\partial x}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial w_{i}}{\partial x}}={\frac {\partial w_{i}}{\partial w_{i-1}}}{\frac {\partial w_{i-1}}{\partial x}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/782624d84a5c71f2ce79128f985771bbf4b932fd" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:21.154ex; height:6.176ex;" alt="{\displaystyle {\frac {\partial w_{i}}{\partial x}}={\frac {\partial w_{i}}{\partial w_{i-1}}}{\frac {\partial w_{i-1}}{\partial x}}}" /> with <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{3}=y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>=</mo> <mi>y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{3}=y}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a497b537bb8ff1f906174a28fb5b4de11fd5adc1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.972ex; height:2.009ex;" alt="{\displaystyle w_{3}=y}" />, and,</li> <li>Reverse accumulation computes the recursive relation: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial y}{\partial w_{i}}}={\frac {\partial y}{\partial w_{i+1}}}{\frac {\partial w_{i+1}}{\partial w_{i}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>+</mo> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial y}{\partial w_{i}}}={\frac {\partial y}{\partial w_{i+1}}}{\frac {\partial w_{i+1}}{\partial w_{i}}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/27ae50b1ca310a206226cf7eec28eecbda406d4b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:21.154ex; height:6.176ex;" alt="{\displaystyle {\frac {\partial y}{\partial w_{i}}}={\frac {\partial y}{\partial w_{i+1}}}{\frac {\partial w_{i+1}}{\partial w_{i}}}}" /> with <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{0}=x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>=</mo> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{0}=x}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1972d70e74c641c859baaae0cb74a4fcdecb70cf" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.147ex; height:2.009ex;" alt="{\displaystyle w_{0}=x}" />.</li></ul> The value of the partial derivative, called the seed, is propagated forward or backward and is initially <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial x}{\partial x}}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial x}{\partial x}}=1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f3545cf2990198da2b54e9d56d57e0ebb707a589" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.005ex; width:7.745ex; height:5.509ex;" alt="{\displaystyle {\frac {\partial x}{\partial x}}=1}" /> or <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial y}{\partial y}}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial y}{\partial y}}=1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/509966af6de3b5822151f71bff1b15fb5528f9ad" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:7.571ex; height:6.176ex;" alt="{\displaystyle {\frac {\partial y}{\partial y}}=1}" />. Forward accumulation evaluates the function and calculates the derivative with respect to one independent variable in one pass. For each independent variable <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{1},x_{2},\dots ,x_{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{1},x_{2},\dots ,x_{n}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/75c2d357bc1b965979bf171b5ba3bac0f68961c2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:13.528ex; height:2.009ex;" alt="{\displaystyle x_{1},x_{2},\dots ,x_{n}}" /> a separate pass is therefore necessary in which the derivative with respect to that independent variable is set to one (<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial x_{1}}{\partial x_{1}}}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial x_{1}}{\partial x_{1}}}=1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3e1dd7f0f410842ef34f483a33a717129f10314b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:8.799ex; height:5.843ex;" alt="{\displaystyle {\frac {\partial x_{1}}{\partial x_{1}}}=1}" />) and of all others to zero (<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial x_{2}}{\partial x_{1}}}=\dots ={\frac {\partial x_{n}}{\partial x_{1}}}=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mo>⋯</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\partial x_{2}}{\partial x_{1}}}=\dots ={\frac {\partial x_{n}}{\partial x_{1}}}=0}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/258410dc722c3d3cd9fbea6ad3f73f76e7d805a7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:22.422ex; height:5.843ex;" alt="{\displaystyle {\frac {\partial x_{2}}{\partial x_{1}}}=\dots ={\frac {\partial x_{n}}{\partial x_{1}}}=0}" />). In contrast, reverse accumulation requires the evaluated partial functions for the partial derivatives. Reverse accumulation therefore evaluates the function first and calculates the derivatives with respect to all independent variables in an additional pass. Which of these two types should be used depends on the sweep count. The <a href="/wiki/Computational_complexity_theory" title="Computational complexity theory">computational complexity</a> of one sweep is proportional to the complexity of the original code. <ul><li>Forward accumulation is more efficient than reverse accumulation for functions f : Rn → Rm with n ≪ m as only n sweeps are necessary, compared to m sweeps for reverse accumulation.</li> <li>Reverse accumulation is more efficient than forward accumulation for functions f : Rn → Rm with n ≫ m as only m sweeps are necessary, compared to n sweeps for forward accumulation.</li></ul> <a href="/wiki/Backpropagation" title="Backpropagation">Backpropagation</a> of errors in multilayer perceptrons, a technique used in <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a>, is a special case of reverse accumulation.<a href="#cite_note-baydin2018automatic-2">[2]</a> Forward accumulation was introduced by R.E. Wengert in 1964.<a href="#cite_note-Wengert1964-13">[13]</a> According to Andreas Griewank, reverse accumulation has been suggested since the late 1960s, but the inventor is unknown.<a href="#cite_note-grie2012-14">[14]</a> <a href="/wiki/Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a> published reverse accumulation in 1976.<a href="#cite_note-lin1976-15">[15]</a> <div class="mw-heading mw-heading3"><h3 id="Forward_accumulation">Forward accumulation</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=6" title="Edit section: Forward accumulation">edit</a>]</div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:ForwardAD.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/6/69/ForwardAD.png/250px-ForwardAD.png" decoding="async" width="220" height="133" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/6/69/ForwardAD.png/330px-ForwardAD.png 1.5x, //upload.wikimedia.org/wikipedia/commons/6/69/ForwardAD.png 2x" data-file-width="439" data-file-height="266" /></a><figcaption>Forward accumulation</figcaption></figure> In forward accumulation AD, one first fixes the independent variable with respect to which differentiation is performed and computes the derivative of each sub-<a href="/wiki/Expression_(mathematics)" title="Expression (mathematics)">expression</a> recursively. In a pen-and-paper calculation, this involves repeatedly substituting the derivative of the inner functions in the chain rule: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}{\frac {\partial y}{\partial x}}&={\frac {\partial y}{\partial w_{n-1}}}{\frac {\partial w_{n-1}}{\partial x}}\\[6pt]&={\frac {\partial y}{\partial w_{n-1}}}\left({\frac {\partial w_{n-1}}{\partial w_{n-2}}}{\frac {\partial w_{n-2}}{\partial x}}\right)\\[6pt]&={\frac {\partial y}{\partial w_{n-1}}}\left({\frac {\partial w_{n-1}}{\partial w_{n-2}}}\left({\frac {\partial w_{n-2}}{\partial w_{n-3}}}{\frac {\partial w_{n-3}}{\partial x}}\right)\right)\\[6pt]&=\cdots \end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="0.9em 0.9em 0.9em 0.3em" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow> <mo>(</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mrow> <mo>)</mo> </mrow> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow> <mo>(</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow> <mo>(</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>3</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>3</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mrow> <mo>)</mo> </mrow> </mrow> <mo>)</mo> </mrow> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mo>⋯</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}{\frac {\partial y}{\partial x}}&={\frac {\partial y}{\partial w_{n-1}}}{\frac {\partial w_{n-1}}{\partial x}}\\[6pt]&={\frac {\partial y}{\partial w_{n-1}}}\left({\frac {\partial w_{n-1}}{\partial w_{n-2}}}{\frac {\partial w_{n-2}}{\partial x}}\right)\\[6pt]&={\frac {\partial y}{\partial w_{n-1}}}\left({\frac {\partial w_{n-1}}{\partial w_{n-2}}}\left({\frac {\partial w_{n-2}}{\partial w_{n-3}}}{\frac {\partial w_{n-3}}{\partial x}}\right)\right)\\[6pt]&=\cdots \end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/cb8633d881482245a77285d008ac3a902276c73c" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -12.338ex; width:43.5ex; height:25.843ex;" alt="{\displaystyle {\begin{aligned}{\frac {\partial y}{\partial x}}&={\frac {\partial y}{\partial w_{n-1}}}{\frac {\partial w_{n-1}}{\partial x}}\\[6pt]&={\frac {\partial y}{\partial w_{n-1}}}\left({\frac {\partial w_{n-1}}{\partial w_{n-2}}}{\frac {\partial w_{n-2}}{\partial x}}\right)\\[6pt]&={\frac {\partial y}{\partial w_{n-1}}}\left({\frac {\partial w_{n-1}}{\partial w_{n-2}}}\left({\frac {\partial w_{n-2}}{\partial w_{n-3}}}{\frac {\partial w_{n-3}}{\partial x}}\right)\right)\\[6pt]&=\cdots \end{aligned}}}" /> This can be generalized to multiple variables as a matrix product of <a href="/wiki/Jacobian_matrix_and_determinant" title="Jacobian matrix and determinant">Jacobians</a>. Compared to reverse accumulation, forward accumulation is natural and easy to implement as the flow of derivative information coincides with the order of evaluation. Each variable <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fe22f0329d3ecb2e1880d44d191aba0e5475db68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" /> is augmented with its derivative <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e4bde19b37f509f1c01409e2a9f63bf3b83fe505" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{i}}" /> (stored as a numerical value, not a symbolic expression), <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{i}={\frac {\partial w_{i}}{\partial x}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{i}={\frac {\partial w_{i}}{\partial x}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/47f37b32a27a024dca990c1835ccf8139e7e3264" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.005ex; width:10.18ex; height:5.509ex;" alt="{\displaystyle {\dot {w}}_{i}={\frac {\partial w_{i}}{\partial x}}}" /> as denoted by the dot. The derivatives are then computed in sync with the evaluation steps and combined with other derivatives via the chain rule. Using the chain rule, if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fe22f0329d3ecb2e1880d44d191aba0e5475db68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" /> has predecessors in the computational graph: <dl><dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{i}=\sum _{j\in \{{\text{predecessors of i}}\}}{\frac {\partial w_{i}}{\partial w_{j}}}{\dot {w}}_{j}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mo>∈</mo> <mo fence="false" stretchy="false">{</mo> <mrow class="MJX-TeXAtom-ORD"> <mtext>predecessors of i</mtext> </mrow> <mo fence="false" stretchy="false">}</mo> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> </mrow> </mfrac> </mrow> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{i}=\sum _{j\in \{{\text{predecessors of i}}\}}{\frac {\partial w_{i}}{\partial w_{j}}}{\dot {w}}_{j}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/96c73bb52bdd6a2a470848340b0d7877d9a1f216" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:28.416ex; height:7.009ex;" alt="{\displaystyle {\dot {w}}_{i}=\sum _{j\in \{{\text{predecessors of i}}\}}{\frac {\partial w_{i}}{\partial w_{j}}}{\dot {w}}_{j}}" /></dd></dl> <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:ForwardAccumulationAutomaticDifferentiation.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/a/a4/ForwardAccumulationAutomaticDifferentiation.png/330px-ForwardAccumulationAutomaticDifferentiation.png" decoding="async" width="300" height="153" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/a/a4/ForwardAccumulationAutomaticDifferentiation.png/500px-ForwardAccumulationAutomaticDifferentiation.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/a/a4/ForwardAccumulationAutomaticDifferentiation.png/960px-ForwardAccumulationAutomaticDifferentiation.png 2x" data-file-width="1346" data-file-height="688" /></a><figcaption>Figure 2: Example of forward accumulation with computational graph</figcaption></figure> As an example, consider the function: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}y&=f(x_{1},x_{2})\\&=x_{1}x_{2}+\sin x_{1}\\&=w_{1}w_{2}+\sin w_{1}\\&=w_{3}+w_{4}\\&=w_{5}\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi>y</mi> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>+</mo> <mi>sin</mi> <mo>⁡</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>+</mo> <mi>sin</mi> <mo>⁡</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>5</mn> </mrow> </msub> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}y&=f(x_{1},x_{2})\\&=x_{1}x_{2}+\sin x_{1}\\&=w_{1}w_{2}+\sin w_{1}\\&=w_{3}+w_{4}\\&=w_{5}\end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d7967705363fb957da83b161d3d9ebacd2856b9c" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -7.005ex; width:19.244ex; height:15.176ex;" alt="{\displaystyle {\begin{aligned}y&=f(x_{1},x_{2})\\&=x_{1}x_{2}+\sin x_{1}\\&=w_{1}w_{2}+\sin w_{1}\\&=w_{3}+w_{4}\\&=w_{5}\end{aligned}}}" /> For clarity, the individual sub-expressions have been labeled with the variables <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fe22f0329d3ecb2e1880d44d191aba0e5475db68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" />. The choice of the independent variable to which differentiation is performed affects the seed values ẇ1 and ẇ2. Given interest in the derivative of this function with respect to x1, the seed values should be set to: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}{\dot {w}}_{1}={\frac {\partial w_{1}}{\partial x_{1}}}={\frac {\partial x_{1}}{\partial x_{1}}}=1\\{\dot {w}}_{2}={\frac {\partial w_{2}}{\partial x_{1}}}={\frac {\partial x_{2}}{\partial x_{1}}}=0\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>1</mn> </mtd> </mtr> <mtr> <mtd> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0</mn> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}{\dot {w}}_{1}={\frac {\partial w_{1}}{\partial x_{1}}}={\frac {\partial x_{1}}{\partial x_{1}}}=1\\{\dot {w}}_{2}={\frac {\partial w_{2}}{\partial x_{1}}}={\frac {\partial x_{2}}{\partial x_{1}}}=0\end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c0466adeb850fe6a565e8b2bc00a43fa1de7b5e5" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -5.338ex; width:23.338ex; height:11.843ex;" alt="{\displaystyle {\begin{aligned}{\dot {w}}_{1}={\frac {\partial w_{1}}{\partial x_{1}}}={\frac {\partial x_{1}}{\partial x_{1}}}=1\\{\dot {w}}_{2}={\frac {\partial w_{2}}{\partial x_{1}}}={\frac {\partial x_{2}}{\partial x_{1}}}=0\end{aligned}}}" /> With the seed values set, the values propagate using the chain rule as shown. Figure 2 shows a pictorial depiction of this process as a computational graph. <dl><dd><table class="wikitable"> <tbody><tr> <th>Operations to compute value</th> <th>Operations to compute derivative </th></tr> <tr> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{1}=x_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>=</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{1}=x_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7f2a49aec3bbb8a6cf5cf6f26f7a8a7fe1899048" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.201ex; height:2.009ex;" alt="{\displaystyle w_{1}=x_{1}}" /></td> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{1}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{1}=1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/06b79aff24809dd6df9ac822b1e67aeb007b52bc" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.979ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{1}=1}" /> (seed) </td></tr> <tr> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{2}=x_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>=</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{2}=x_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c21123bc68d621265d8643ed990ee0c75989f5bd" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.201ex; height:2.009ex;" alt="{\displaystyle w_{2}=x_{2}}" /></td> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{2}=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{2}=0}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1b9af79f3a9111228543209a98a0c22065e35d50" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.979ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{2}=0}" /> (seed) </td></tr> <tr> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{3}=w_{1}\cdot w_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>⋅</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{3}=w_{1}\cdot w_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6322e282f9ab4ccfd700413eb9e6b7f7ebd18614" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:12.933ex; height:2.009ex;" alt="{\displaystyle w_{3}=w_{1}\cdot w_{2}}" /></td> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{3}=w_{2}\cdot {\dot {w}}_{1}+w_{1}\cdot {\dot {w}}_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>⋅</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>⋅</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{3}=w_{2}\cdot {\dot {w}}_{1}+w_{1}\cdot {\dot {w}}_{2}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1d471df3cbe04205a77956aafef33308e903ba8c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:22.889ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{3}=w_{2}\cdot {\dot {w}}_{1}+w_{1}\cdot {\dot {w}}_{2}}" /> </td></tr> <tr> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{4}=\sin w_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> <mo>=</mo> <mi>sin</mi> <mo>⁡</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{4}=\sin w_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/cfd8dda3fcca754a322a692bff3259fc760141d1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.778ex; height:2.509ex;" alt="{\displaystyle w_{4}=\sin w_{1}}" /></td> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{4}=\cos w_{1}\cdot {\dot {w}}_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> <mo>=</mo> <mi>cos</mi> <mo>⁡</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>⋅</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{4}=\cos w_{1}\cdot {\dot {w}}_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e6a3417c4b7011e60e980b4b081b74ce6f6904d3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:16.431ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{4}=\cos w_{1}\cdot {\dot {w}}_{1}}" /> </td></tr> <tr> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{5}=w_{3}+w_{4}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>5</mn> </mrow> </msub> <mo>=</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{5}=w_{3}+w_{4}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/758fb1c0311683a854434b392138444c81b343f0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:14.094ex; height:2.343ex;" alt="{\displaystyle w_{5}=w_{3}+w_{4}}" /></td> <td><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{5}={\dot {w}}_{3}+{\dot {w}}_{4}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>5</mn> </mrow> </msub> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>+</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{5}={\dot {w}}_{3}+{\dot {w}}_{4}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3b04661f38fdce46e4451d7486d394661c811263" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:14.094ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{5}={\dot {w}}_{3}+{\dot {w}}_{4}}" /> </td></tr></tbody></table></dd></dl> To compute the <a href="/wiki/Gradient" title="Gradient">gradient</a> of this example function, which requires not only <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tfrac {\partial y}{\partial x_{1}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mstyle> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\tfrac {\partial y}{\partial x_{1}}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/76674bc0958f54ec6744032a86c1a4e591563370" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.671ex; width:3.54ex; height:4.509ex;" alt="{\displaystyle {\tfrac {\partial y}{\partial x_{1}}}}" /> but also <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tfrac {\partial y}{\partial x_{2}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mstyle> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\tfrac {\partial y}{\partial x_{2}}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/5d269239696326c4d212efdb58f0fa5c1f7f3bd0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.671ex; width:3.54ex; height:4.509ex;" alt="{\displaystyle {\tfrac {\partial y}{\partial x_{2}}}}" />, an additional sweep is performed over the computational graph using the seed values <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\dot {w}}_{1}=0;{\dot {w}}_{2}=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>=</mo> <mn>0</mn> <mo>;</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo>˙</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\dot {w}}_{1}=0;{\dot {w}}_{2}=1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/95e60a42770738168db349827e4e2078e5b5e3ca" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:14.993ex; height:2.509ex;" alt="{\displaystyle {\dot {w}}_{1}=0;{\dot {w}}_{2}=1}" />. <div class="mw-heading mw-heading4"><h4 id="Implementation">Implementation</h4>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=7" title="Edit section: Implementation">edit</a>]</div> <div class="mw-heading mw-heading5"><h5 id="Pseudocode">Pseudocode</h5>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=8" title="Edit section: Pseudocode">edit</a>]</div> Forward accumulation calculates the function and the derivative (but only for one independent variable each) in one pass. The associated method call expects the expression Z to be derived with regard to a variable V. The method returns a pair of the evaluated function and its derivative. The method traverses the expression tree recursively until a variable is reached. If the derivative with respect to this variable is requested, its derivative is 1, 0 otherwise. Then the partial function as well as the partial derivative are evaluated.<a href="#cite_note-demm22-16">[16]</a> <div class="mw-highlight mw-highlight-lang-cpp mw-content-ltr" dir="ltr"><pre>tuple<float,float> evaluateAndDerive(Expression Z, Variable V) { if isVariable(Z) if (Z = V) return {valueOf(Z), 1}; else return {valueOf(Z), 0}; else if (Z = A + B) {a, a'} = evaluateAndDerive(A, V); {b, b'} = evaluateAndDerive(B, V); return {a + b, a' + b'}; else if (Z = A - B) {a, a'} = evaluateAndDerive(A, V); {b, b'} = evaluateAndDerive(B, V); return {a - b, a' - b'}; else if (Z = A * B) {a, a'} = evaluateAndDerive(A, V); {b, b'} = evaluateAndDerive(B, V); return {a * b, b * a' + a * b'}; } </pre></div> <div class="mw-heading mw-heading5"><h5 id="C++">C++</h5>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=9" title="Edit section: C++">edit</a>]</div> <div class="mw-highlight mw-highlight-lang-cpp mw-content-ltr" dir="ltr"><pre>#include <iostream> struct ValueAndPartial { float value, partial; }; struct Variable; struct Expression { virtual ValueAndPartial evaluateAndDerive(Variable *variable) = 0; }; struct Variable: public Expression { float value; Variable(float value): value(value) {} ValueAndPartial evaluateAndDerive(Variable *variable) { float partial = (this == variable) ? 1.0f : 0.0f; return {value, partial}; } }; struct Plus: public Expression { Expression *a, *b; Plus(Expression *a, Expression *b): a(a), b(b) {} ValueAndPartial evaluateAndDerive(Variable *variable) { auto [valueA, partialA] = a->evaluateAndDerive(variable); auto [valueB, partialB] = b->evaluateAndDerive(variable); return {valueA + valueB, partialA + partialB}; } }; struct Multiply: public Expression { Expression *a, *b; Multiply(Expression *a, Expression *b): a(a), b(b) {} ValueAndPartial evaluateAndDerive(Variable *variable) { auto [valueA, partialA] = a->evaluateAndDerive(variable); auto [valueB, partialB] = b->evaluateAndDerive(variable); return {valueA * valueB, valueB * partialA + valueA * partialB}; } }; int main () { // Example: Finding the partials of z = x * (x + y) + y * y at (x, y) = (2, 3) Variable x(2), y(3); Plus p1(&x, &y); Multiply m1(&x, &p1); Multiply m2(&y, &y); Plus z(&m1, &m2); float xPartial = z.evaluateAndDerive(&x).partial; float yPartial = z.evaluateAndDerive(&y).partial; std::cout << "∂z/∂x = " << xPartial << ", " << "∂z/∂y = " << yPartial << std::endl; // Output: ∂z/∂x = 7, ∂z/∂y = 8 return 0; } </pre></div> <div class="mw-heading mw-heading3"><h3 id="Reverse_accumulation">Reverse accumulation</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=10" title="Edit section: Reverse accumulation">edit</a>]</div> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:AutoDiff.webp" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/c/c5/AutoDiff.webp/220px-AutoDiff.webp.png" decoding="async" width="220" height="79" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/c/c5/AutoDiff.webp/330px-AutoDiff.webp.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/c/c5/AutoDiff.webp/440px-AutoDiff.webp.png 2x" data-file-width="911" data-file-height="328" /></a><figcaption>Reverse accumulation</figcaption></figure> In reverse accumulation AD, the dependent variable to be differentiated is fixed and the derivative is computed with respect to each sub-<a href="/wiki/Expression_(mathematics)" title="Expression (mathematics)">expression</a> recursively. In a pen-and-paper calculation, the derivative of the outer functions is repeatedly substituted in the chain rule: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}{\frac {\partial y}{\partial x}}&={\frac {\partial y}{\partial w_{1}}}{\frac {\partial w_{1}}{\partial x}}\\&=\left({\frac {\partial y}{\partial w_{2}}}{\frac {\partial w_{2}}{\partial w_{1}}}\right){\frac {\partial w_{1}}{\partial x}}\\&=\left(\left({\frac {\partial y}{\partial w_{3}}}{\frac {\partial w_{3}}{\partial w_{2}}}\right){\frac {\partial w_{2}}{\partial w_{1}}}\right){\frac {\partial w_{1}}{\partial x}}\\&=\cdots \end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>(</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> </mrow> <mo>)</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>(</mo> <mrow> <mrow> <mo>(</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> </mrow> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> </mfrac> </mrow> </mrow> <mo>)</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> </mfrac> </mrow> </mrow> <mo>)</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <mi>x</mi> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mo>⋯</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}{\frac {\partial y}{\partial x}}&={\frac {\partial y}{\partial w_{1}}}{\frac {\partial w_{1}}{\partial x}}\\&=\left({\frac {\partial y}{\partial w_{2}}}{\frac {\partial w_{2}}{\partial w_{1}}}\right){\frac {\partial w_{1}}{\partial x}}\\&=\left(\left({\frac {\partial y}{\partial w_{3}}}{\frac {\partial w_{3}}{\partial w_{2}}}\right){\frac {\partial w_{2}}{\partial w_{1}}}\right){\frac {\partial w_{1}}{\partial x}}\\&=\cdots \end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/16bec7d55f27946c3dbe4e1e8eee9e390b31b7c9" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -10.171ex; width:34.441ex; height:21.509ex;" alt="{\displaystyle {\begin{aligned}{\frac {\partial y}{\partial x}}&={\frac {\partial y}{\partial w_{1}}}{\frac {\partial w_{1}}{\partial x}}\\&=\left({\frac {\partial y}{\partial w_{2}}}{\frac {\partial w_{2}}{\partial w_{1}}}\right){\frac {\partial w_{1}}{\partial x}}\\&=\left(\left({\frac {\partial y}{\partial w_{3}}}{\frac {\partial w_{3}}{\partial w_{2}}}\right){\frac {\partial w_{2}}{\partial w_{1}}}\right){\frac {\partial w_{1}}{\partial x}}\\&=\cdots \end{aligned}}}" /> In reverse accumulation, the quantity of interest is the adjoint, denoted with a bar <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1f4fb533e150ad8a9869ca843408fb98d3e87221" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.343ex;" alt="{\displaystyle {\bar {w}}_{i}}" />; it is a derivative of a chosen dependent variable with respect to a subexpression <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fe22f0329d3ecb2e1880d44d191aba0e5475db68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" />: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{i}={\frac {\partial y}{\partial w_{i}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <mi>y</mi> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{i}={\frac {\partial y}{\partial w_{i}}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a0116dede5ca0d81c0e41a9b62c2522e4f091d8a" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:10.18ex; height:6.009ex;" alt="{\displaystyle {\bar {w}}_{i}={\frac {\partial y}{\partial w_{i}}}}" /> Using the chain rule, if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{i}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fe22f0329d3ecb2e1880d44d191aba0e5475db68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.464ex; height:2.009ex;" alt="{\displaystyle w_{i}}" /> has successors in the computational graph: <dl><dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{i}=\sum _{j\in \{{\text{successors of i}}\}}{\bar {w}}_{j}{\frac {\partial w_{j}}{\partial w_{i}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mo>∈</mo> <mo fence="false" stretchy="false">{</mo> <mrow class="MJX-TeXAtom-ORD"> <mtext>successors of i</mtext> </mrow> <mo fence="false" stretchy="false">}</mo> </mrow> </munder> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> </mrow> </msub> </mrow> <mrow> <mi mathvariant="normal">∂</mi> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{i}=\sum _{j\in \{{\text{successors of i}}\}}{\bar {w}}_{j}{\frac {\partial w_{j}}{\partial w_{i}}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e5986463d8b2e48d0da5233099bb97bc4ea89844" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:26.775ex; height:7.343ex;" alt="{\displaystyle {\bar {w}}_{i}=\sum _{j\in \{{\text{successors of i}}\}}{\bar {w}}_{j}{\frac {\partial w_{j}}{\partial w_{i}}}}" /></dd></dl> Reverse accumulation traverses the chain rule from outside to inside, or in the case of the computational graph in Figure 3, from top to bottom. The example function is scalar-valued, and thus there is only one seed for the derivative computation, and only one sweep of the computational graph is needed to calculate the (two-component) gradient. This is only <a href="/wiki/Space%E2%80%93time_tradeoff" title="Space–time tradeoff">half the work</a> when compared to forward accumulation, but reverse accumulation requires the storage of the intermediate variables wi as well as the instructions that produced them in a data structure known as a "tape" or a Wengert list<a href="#cite_note-17">[17]</a> (however, Wengert published forward accumulation, not reverse accumulation<a href="#cite_note-Wengert1964-13">[13]</a>), which may consume significant memory if the computational graph is large. This can be mitigated to some extent by storing only a subset of the intermediate variables and then reconstructing the necessary work variables by repeating the evaluations, a technique known as <a href="/wiki/Rematerialization" title="Rematerialization">rematerialization</a>. <a href="/wiki/Checkpointing_scheme" title="Checkpointing scheme">Checkpointing</a> is also used to save intermediary states. <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:ReverseaccumulationAD.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/a/a0/ReverseaccumulationAD.png/330px-ReverseaccumulationAD.png" decoding="async" width="300" height="175" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/a/a0/ReverseaccumulationAD.png/500px-ReverseaccumulationAD.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/a/a0/ReverseaccumulationAD.png/960px-ReverseaccumulationAD.png 2x" data-file-width="1319" data-file-height="769" /></a><figcaption>Figure 3: Example of reverse accumulation with computational graph</figcaption></figure> The operations to compute the derivative using reverse accumulation are shown in the table below (note the reversed order): <style data-mw-deduplicate="TemplateStyles:r996643573">.mw-parser-output .block-indent{padding-left:3em;padding-right:0;overflow:hidden}</style><div class="block-indent"> <dl><dt>Operations to compute derivative</dt> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{5}=1{\text{ (seed)}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>5</mn> </mrow> </msub> <mo>=</mo> <mn>1</mn> <mrow class="MJX-TeXAtom-ORD"> <mtext> (seed)</mtext> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{5}=1{\text{ (seed)}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6db8ede325fd22d2a743b211384104aab821d982" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.643ex; height:2.843ex;" alt="{\displaystyle {\bar {w}}_{5}=1{\text{ (seed)}}}" /></dd> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{4}={\bar {w}}_{5}\cdot 1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>5</mn> </mrow> </msub> <mo>⋅</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{4}={\bar {w}}_{5}\cdot 1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/13fbd3942c5c3681829c14591fa0283615e1d37b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.377ex; height:2.509ex;" alt="{\displaystyle {\bar {w}}_{4}={\bar {w}}_{5}\cdot 1}" /></dd> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{3}={\bar {w}}_{5}\cdot 1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>5</mn> </mrow> </msub> <mo>⋅</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{3}={\bar {w}}_{5}\cdot 1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089a2458f30e2878986bbf37b6d9e6ff05b04eda" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.377ex; height:2.509ex;" alt="{\displaystyle {\bar {w}}_{3}={\bar {w}}_{5}\cdot 1}" /></dd> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{2}={\bar {w}}_{3}\cdot w_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>⋅</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{2}={\bar {w}}_{3}\cdot w_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/caee22b3e118230264399fcefe86cb3c65ef5a95" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:12.933ex; height:2.343ex;" alt="{\displaystyle {\bar {w}}_{2}={\bar {w}}_{3}\cdot w_{1}}" /></dd> <dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {w}}_{1}={\bar {w}}_{3}\cdot w_{2}+{\bar {w}}_{4}\cdot \cos w_{1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>3</mn> </mrow> </msub> <mo>⋅</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>+</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>w</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>4</mn> </mrow> </msub> <mo>⋅</mo> <mi>cos</mi> <mo>⁡</mo> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {w}}_{1}={\bar {w}}_{3}\cdot w_{2}+{\bar {w}}_{4}\cdot \cos w_{1}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/cfe9ea5297733fa81122ea52669cfb41437ab7af" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:26.387ex; height:2.343ex;" alt="{\displaystyle {\bar {w}}_{1}={\bar {w}}_{3}\cdot w_{2}+{\bar {w}}_{4}\cdot \cos w_{1}}" /></dd></dl> </div> The data flow graph of a computation can be manipulated to calculate the gradient of its original calculation. This is done by adding an adjoint node for each primal node, connected by adjoint edges which parallel the primal edges but flow in the opposite direction. The nodes in the adjoint graph represent multiplication by the derivatives of the functions calculated by the nodes in the primal. For instance, addition in the primal causes fanout in the adjoint; fanout in the primal causes addition in the adjoint;<a href="#cite_note-18">[a]</a> a <a href="/wiki/Unary_operation" title="Unary operation">unary</a> function y = f(x) in the primal causes x̄ = ȳ f′(x) in the adjoint; etc. <div class="mw-heading mw-heading4"><h4 id="Implementation_2">Implementation</h4>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=11" title="Edit section: Implementation">edit</a>]</div> <div class="mw-heading mw-heading5"><h5 id="Pseudo_code">Pseudo code</h5>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=12" title="Edit section: Pseudo code">edit</a>]</div> Reverse accumulation requires two passes: In the forward pass, the function is evaluated first and the partial results are cached. In the reverse pass, the partial derivatives are calculated and the previously derived value is backpropagated. The corresponding method call expects the expression Z to be derived and seeded with the derived value of the parent expression. For the top expression, Z differentiated with respect to Z, this is 1. The method traverses the expression tree recursively until a variable is reached and adds the current seed value to the derivative expression.<a href="#cite_note-ssdbm21-19">[18]</a><a href="#cite_note-dpd-20">[19]</a> <div class="mw-highlight mw-highlight-lang-cpp mw-content-ltr" dir="ltr"><pre>void derive(Expression Z, float seed) { if isVariable(Z) partialDerivativeOf(Z) += seed; else if (Z = A + B) derive(A, seed); derive(B, seed); else if (Z = A - B) derive(A, seed); derive(B, -seed); else if (Z = A * B) derive(A, valueOf(B) * seed); derive(B, valueOf(A) * seed); } </pre></div> <div class="mw-heading mw-heading5"><h5 id="C++_2">C++</h5>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=13" title="Edit section: C++">edit</a>]</div> <div class="mw-highlight mw-highlight-lang-cpp mw-content-ltr" dir="ltr"><pre>#include <iostream> struct Expression { float value; virtual void evaluate() = 0; virtual void derive(float seed) = 0; }; struct Variable: public Expression { float partial; Variable(float value) { this->value = value; partial = 0.0f; } void evaluate() {} void derive(float seed) { partial += seed; } }; struct Plus: public Expression { Expression *a, *b; Plus(Expression *a, Expression *b): a(a), b(b) {} void evaluate() { a->evaluate(); b->evaluate(); value = a->value + b->value; } void derive(float seed) { a->derive(seed); b->derive(seed); } }; struct Multiply: public Expression { Expression *a, *b; Multiply(Expression *a, Expression *b): a(a), b(b) {} void evaluate() { a->evaluate(); b->evaluate(); value = a->value * b->value; } void derive(float seed) { a->derive(b->value * seed); b->derive(a->value * seed); } }; int main () { // Example: Finding the partials of z = x * (x + y) + y * y at (x, y) = (2, 3) Variable x(2), y(3); Plus p1(&x, &y); Multiply m1(&x, &p1); Multiply m2(&y, &y); Plus z(&m1, &m2); z.evaluate(); std::cout << "z = " << z.value << std::endl; // Output: z = 19 z.derive(1); std::cout << "∂z/∂x = " << x.partial << ", " << "∂z/∂y = " << y.partial << std::endl; // Output: ∂z/∂x = 7, ∂z/∂y = 8 return 0; } </pre></div> <div class="mw-heading mw-heading3"><h3 id="Beyond_forward_and_reverse_accumulation">Beyond forward and reverse accumulation</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=14" title="Edit section: Beyond forward and reverse accumulation">edit</a>]</div> Forward and reverse accumulation are just two (extreme) ways of traversing the chain rule. The problem of computing a full Jacobian of f : Rn → Rm with a minimum number of arithmetic operations is known as the optimal Jacobian accumulation (OJA) problem, which is <a href="/wiki/NP-complete" class="mw-redirect" title="NP-complete">NP-complete</a>.<a href="#cite_note-21">[20]</a> Central to this proof is the idea that algebraic dependencies may exist between the local partials that label the edges of the graph. In particular, two or more edge labels may be recognized as equal. The complexity of the problem is still open if it is assumed that all edge labels are unique and algebraically independent. <div class="mw-heading mw-heading2"><h2 id="Automatic_differentiation_using_dual_numbers">Automatic differentiation using dual numbers</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=15" title="Edit section: Automatic differentiation using dual numbers">edit</a>]</div> Forward mode automatic differentiation is accomplished by augmenting the <a href="/wiki/Algebra_over_a_field" title="Algebra over a field">algebra</a> of <a href="/wiki/Real_numbers" class="mw-redirect" title="Real numbers">real numbers</a> and obtaining a new <a href="/wiki/Arithmetic" title="Arithmetic">arithmetic</a>. An additional component is added to every number to represent the derivative of a function at the number, and all arithmetic operators are extended for the augmented algebra. The augmented algebra is the algebra of <a href="/wiki/Dual_numbers" class="mw-redirect" title="Dual numbers">dual numbers</a>. Replace every number <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \,x}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mspace width="thinmathspace"></mspace> <mi>x</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \,x}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/85e12927a3ef8b1d7463100e34779c2761ad4765" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.717ex; height:1.676ex;" alt="{\displaystyle \,x}" /> with the number <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x+x'\varepsilon }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x+x'\varepsilon }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/596f8af3617fcc910c2d3f48542044837a843213" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:7.268ex; height:2.676ex;" alt="{\displaystyle x+x'\varepsilon }" />, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x'}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>x</mi> <mo>′</mo> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x'}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0ac74959896052e160a5953102e4bc3850fe93b2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.014ex; height:2.509ex;" alt="{\displaystyle x'}" /> is a real number, but <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \varepsilon }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>ε</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \varepsilon }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a30c89172e5b88edbd45d3e2772c7f5e562e5173" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.083ex; height:1.676ex;" alt="{\displaystyle \varepsilon }" /> is an <a href="/wiki/Abstract_number" class="mw-redirect" title="Abstract number">abstract number</a> with the property <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \varepsilon ^{2}=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \varepsilon ^{2}=0}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68eddcca75ba10af115dde98b267c3afd5341d40" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.399ex; height:2.676ex;" alt="{\displaystyle \varepsilon ^{2}=0}" /> (an <a href="/wiki/Infinitesimal" title="Infinitesimal">infinitesimal</a>; see <a href="/wiki/Smooth_infinitesimal_analysis" title="Smooth infinitesimal analysis">Smooth infinitesimal analysis</a>). Using only this, regular arithmetic gives <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}(x+x'\varepsilon )+(y+y'\varepsilon )&=x+y+(x'+y')\varepsilon \\(x+x'\varepsilon )-(y+y'\varepsilon )&=x-y+(x'-y')\varepsilon \\(x+x'\varepsilon )\cdot (y+y'\varepsilon )&=xy+xy'\varepsilon +yx'\varepsilon +x'y'\varepsilon ^{2}=xy+(xy'+yx')\varepsilon \\(x+x'\varepsilon )/(y+y'\varepsilon )&=(x/y+x'\varepsilon /y)/(1+y'\varepsilon /y)=(x/y+x'\varepsilon /y)\cdot (1-y'\varepsilon /y)=x/y+(x'/y-xy'/y^{2})\varepsilon \end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mo stretchy="false">(</mo> <mi>y</mi> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>x</mi> <mo>+</mo> <mi>y</mi> <mo>+</mo> <mo stretchy="false">(</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mo stretchy="false">)</mo> <mi>ε</mi> </mtd> </mtr> <mtr> <mtd> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mo stretchy="false">(</mo> <mi>y</mi> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>x</mi> <mo>−</mo> <mi>y</mi> <mo>+</mo> <mo stretchy="false">(</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo>−</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mo stretchy="false">)</mo> <mi>ε</mi> </mtd> </mtr> <mtr> <mtd> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <mo stretchy="false">(</mo> <mi>y</mi> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mi>x</mi> <mi>y</mi> <mo>+</mo> <mi>x</mi> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo>+</mo> <mi>y</mi> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <msup> <mi>y</mi> <mo>′</mo> </msup> <msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> <mo>=</mo> <mi>x</mi> <mi>y</mi> <mo>+</mo> <mo stretchy="false">(</mo> <mi>x</mi> <msup> <mi>y</mi> <mo>′</mo> </msup> <mo>+</mo> <mi>y</mi> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo stretchy="false">)</mo> <mi>ε</mi> </mtd> </mtr> <mtr> <mtd> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mo stretchy="false">(</mo> <mi>y</mi> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mo stretchy="false">(</mo> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mo stretchy="false">(</mo> <mn>1</mn> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mo stretchy="false">(</mo> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo>+</mo> <mo stretchy="false">(</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo>−</mo> <mi>x</mi> <msup> <mi>y</mi> <mo>′</mo> </msup> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <msup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> <mo stretchy="false">)</mo> <mi>ε</mi> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}(x+x'\varepsilon )+(y+y'\varepsilon )&=x+y+(x'+y')\varepsilon \\(x+x'\varepsilon )-(y+y'\varepsilon )&=x-y+(x'-y')\varepsilon \\(x+x'\varepsilon )\cdot (y+y'\varepsilon )&=xy+xy'\varepsilon +yx'\varepsilon +x'y'\varepsilon ^{2}=xy+(xy'+yx')\varepsilon \\(x+x'\varepsilon )/(y+y'\varepsilon )&=(x/y+x'\varepsilon /y)/(1+y'\varepsilon /y)=(x/y+x'\varepsilon /y)\cdot (1-y'\varepsilon /y)=x/y+(x'/y-xy'/y^{2})\varepsilon \end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d64ae677bfbef045cf69a89fbea209a9d72c81af" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -5.838ex; width:106.189ex; height:12.843ex;" alt="{\displaystyle {\begin{aligned}(x+x'\varepsilon )+(y+y'\varepsilon )&=x+y+(x'+y')\varepsilon \\(x+x'\varepsilon )-(y+y'\varepsilon )&=x-y+(x'-y')\varepsilon \\(x+x'\varepsilon )\cdot (y+y'\varepsilon )&=xy+xy'\varepsilon +yx'\varepsilon +x'y'\varepsilon ^{2}=xy+(xy'+yx')\varepsilon \\(x+x'\varepsilon )/(y+y'\varepsilon )&=(x/y+x'\varepsilon /y)/(1+y'\varepsilon /y)=(x/y+x'\varepsilon /y)\cdot (1-y'\varepsilon /y)=x/y+(x'/y-xy'/y^{2})\varepsilon \end{aligned}}}" /> using <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (1+y'\varepsilon /y)\cdot (1-y'\varepsilon /y)=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mn>1</mn> <mo>+</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <msup> <mi>y</mi> <mo>′</mo> </msup> <mi>ε</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (1+y'\varepsilon /y)\cdot (1-y'\varepsilon /y)=1}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d15fb69a99bfa1812549ca4098b16b54a5e22f3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:28.058ex; height:3.009ex;" alt="{\displaystyle (1+y'\varepsilon /y)\cdot (1-y'\varepsilon /y)=1}" />. Now, <a href="/wiki/Polynomials" class="mw-redirect" title="Polynomials">polynomials</a> can be calculated in this augmented arithmetic. If <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P(x)=p_{0}+p_{1}x+p_{2}x^{2}+\cdots +p_{n}x^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>P</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mi>x</mi> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <msup> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> <mo>+</mo> <mo>⋯</mo> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <msup> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P(x)=p_{0}+p_{1}x+p_{2}x^{2}+\cdots +p_{n}x^{n}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fef9fe157d530d7511f9b825fde9fe7fa98ef1d9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:37.388ex; height:3.176ex;" alt="{\displaystyle P(x)=p_{0}+p_{1}x+p_{2}x^{2}+\cdots +p_{n}x^{n}}" />, then <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}P(x+x'\varepsilon )&=p_{0}+p_{1}(x+x'\varepsilon )+\cdots +p_{n}(x+x'\varepsilon )^{n}\\&=p_{0}+p_{1}x+\cdots +p_{n}x^{n}+p_{1}x'\varepsilon +2p_{2}xx'\varepsilon +\cdots +np_{n}x^{n-1}x'\varepsilon \\&=P(x)+P^{(1)}(x)x'\varepsilon \end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi>P</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mo>⋯</mo> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>+</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <msup> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mi>x</mi> <mo>+</mo> <mo>⋯</mo> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <msup> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo>+</mo> <mn>2</mn> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mi>x</mi> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> <mo>+</mo> <mo>⋯</mo> <mo>+</mo> <mi>n</mi> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <msup> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msup> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> </mtd> </mtr> <mtr> <mtd></mtd> <mtd> <mi></mi> <mo>=</mo> <mi>P</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>+</mo> <msup> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mn>1</mn> <mo stretchy="false">)</mo> </mrow> </msup> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mi>ε</mi> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}P(x+x'\varepsilon )&=p_{0}+p_{1}(x+x'\varepsilon )+\cdots +p_{n}(x+x'\varepsilon )^{n}\\&=p_{0}+p_{1}x+\cdots +p_{n}x^{n}+p_{1}x'\varepsilon +2p_{2}xx'\varepsilon +\cdots +np_{n}x^{n-1}x'\varepsilon \\&=P(x)+P^{(1)}(x)x'\varepsilon \end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/5da374c9751959903e867f99e44f88d493841a38" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -4.171ex; width:75.379ex; height:9.509ex;" alt="{\displaystyle {\begin{aligned}P(x+x'\varepsilon )&=p_{0}+p_{1}(x+x'\varepsilon )+\cdots +p_{n}(x+x'\varepsilon )^{n}\\&=p_{0}+p_{1}x+\cdots +p_{n}x^{n}+p_{1}x'\varepsilon +2p_{2}xx'\varepsilon +\cdots +np_{n}x^{n-1}x'\varepsilon \\&=P(x)+P^{(1)}(x)x'\varepsilon \end{aligned}}}" /> where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P^{(1)}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mn>1</mn> <mo stretchy="false">)</mo> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P^{(1)}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3b4fa9f0d7f96845fe125af19bd10f14fa19b854" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:4.155ex; height:2.843ex;" alt="{\displaystyle P^{(1)}}" /> denotes the derivative of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>P</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b4dc73bf40314945ff376bd363916a738548d40a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.745ex; height:2.176ex;" alt="{\displaystyle P}" /> with respect to its first argument, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x'}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>x</mi> <mo>′</mo> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x'}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0ac74959896052e160a5953102e4bc3850fe93b2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.014ex; height:2.509ex;" alt="{\displaystyle x'}" />, called a seed, can be chosen arbitrarily. The new arithmetic consists of <a href="/wiki/Ordered_pair" title="Ordered pair">ordered pairs</a>, elements written <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \langle x,x'\rangle }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">⟨</mo> <mi>x</mi> <mo>,</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo fence="false" stretchy="false">⟩</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \langle x,x'\rangle }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b7b3cc993b372ce43ef82fe69192a33bb2d03520" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.187ex; height:3.009ex;" alt="{\displaystyle \langle x,x'\rangle }" />, with ordinary arithmetics on the first component, and first order differentiation arithmetic on the second component, as described above. Extending the above results on polynomials to <a href="/wiki/Analytic_functions" class="mw-redirect" title="Analytic functions">analytic functions</a> gives a list of the basic arithmetic and some standard functions for the new arithmetic: <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}\left\langle u,u'\right\rangle +\left\langle v,v'\right\rangle &=\left\langle u+v,u'+v'\right\rangle \\\left\langle u,u'\right\rangle -\left\langle v,v'\right\rangle &=\left\langle u-v,u'-v'\right\rangle \\\left\langle u,u'\right\rangle *\left\langle v,v'\right\rangle &=\left\langle uv,u'v+uv'\right\rangle \\\left\langle u,u'\right\rangle /\left\langle v,v'\right\rangle &=\left\langle {\frac {u}{v}},{\frac {u'v-uv'}{v^{2}}}\right\rangle \quad (v\neq 0)\\\sin \left\langle u,u'\right\rangle &=\left\langle \sin(u),u'\cos(u)\right\rangle \\\cos \left\langle u,u'\right\rangle &=\left\langle \cos(u),-u'\sin(u)\right\rangle \\\exp \left\langle u,u'\right\rangle &=\left\langle \exp u,u'\exp u\right\rangle \\\log \left\langle u,u'\right\rangle &=\left\langle \log(u),u'/u\right\rangle \quad (u>0)\\\left\langle u,u'\right\rangle ^{k}&=\left\langle u^{k},u'ku^{k-1}\right\rangle \quad (u\neq 0)\\\left|\left\langle u,u'\right\rangle \right|&=\left\langle \left|u\right|,u'\operatorname {sign} u\right\rangle \quad (u\neq 0)\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> <mo>+</mo> <mrow> <mo>⟨</mo> <mrow> <mi>v</mi> <mo>,</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>+</mo> <mi>v</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mo>+</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> </mtr> <mtr> <mtd> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> <mo>−</mo> <mrow> <mo>⟨</mo> <mrow> <mi>v</mi> <mo>,</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>−</mo> <mi>v</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mo>−</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> </mtr> <mtr> <mtd> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> <mo>∗</mo> <mrow> <mo>⟨</mo> <mrow> <mi>v</mi> <mo>,</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mi>v</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>v</mi> <mo>+</mo> <mi>u</mi> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> </mtr> <mtr> <mtd> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mrow> <mo>⟨</mo> <mrow> <mi>v</mi> <mo>,</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mi>u</mi> <mi>v</mi> </mfrac> </mrow> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>v</mi> <mo>−</mo> <mi>u</mi> <msup> <mi>v</mi> <mo>′</mo> </msup> </mrow> <msup> <mi>v</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mfrac> </mrow> </mrow> <mo>⟩</mo> </mrow> <mspace width="1em"></mspace> <mo stretchy="false">(</mo> <mi>v</mi> <mo>≠</mo> <mn>0</mn> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <mi>sin</mi> <mo>⁡</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>sin</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>u</mi> <mo stretchy="false">)</mo> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>cos</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>u</mi> <mo stretchy="false">)</mo> </mrow> <mo>⟩</mo> </mrow> </mtd> </mtr> <mtr> <mtd> <mi>cos</mi> <mo>⁡</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>cos</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>u</mi> <mo stretchy="false">)</mo> <mo>,</mo> <mo>−</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>sin</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>u</mi> <mo stretchy="false">)</mo> </mrow> <mo>⟩</mo> </mrow> </mtd> </mtr> <mtr> <mtd> <mi>exp</mi> <mo>⁡</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>exp</mi> <mo>⁡</mo> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>exp</mi> <mo>⁡</mo> <mi>u</mi> </mrow> <mo>⟩</mo> </mrow> </mtd> </mtr> <mtr> <mtd> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mi>log</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>u</mi> <mo stretchy="false">)</mo> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mi>u</mi> </mrow> <mo>⟩</mo> </mrow> <mspace width="1em"></mspace> <mo stretchy="false">(</mo> <mi>u</mi> <mo>></mo> <mn>0</mn> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <msup> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msup> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <msup> <mi>u</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msup> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>k</mi> <msup> <mi>u</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mo>−</mo> <mn>1</mn> </mrow> </msup> </mrow> <mo>⟩</mo> </mrow> <mspace width="1em"></mspace> <mo stretchy="false">(</mo> <mi>u</mi> <mo>≠</mo> <mn>0</mn> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd> <mrow> <mo>|</mo> <mrow> <mo>⟨</mo> <mrow> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> </mrow> <mo>⟩</mo> </mrow> <mo>|</mo> </mrow> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow> <mo>⟨</mo> <mrow> <mrow> <mo>|</mo> <mi>u</mi> <mo>|</mo> </mrow> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mi>sign</mi> <mo>⁡</mo> <mi>u</mi> </mrow> <mo>⟩</mo> </mrow> <mspace width="1em"></mspace> <mo stretchy="false">(</mo> <mi>u</mi> <mo>≠</mo> <mn>0</mn> <mo stretchy="false">)</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\left\langle u,u'\right\rangle +\left\langle v,v'\right\rangle &=\left\langle u+v,u'+v'\right\rangle \\\left\langle u,u'\right\rangle -\left\langle v,v'\right\rangle &=\left\langle u-v,u'-v'\right\rangle \\\left\langle u,u'\right\rangle *\left\langle v,v'\right\rangle &=\left\langle uv,u'v+uv'\right\rangle \\\left\langle u,u'\right\rangle /\left\langle v,v'\right\rangle &=\left\langle {\frac {u}{v}},{\frac {u'v-uv'}{v^{2}}}\right\rangle \quad (v\neq 0)\\\sin \left\langle u,u'\right\rangle &=\left\langle \sin(u),u'\cos(u)\right\rangle \\\cos \left\langle u,u'\right\rangle &=\left\langle \cos(u),-u'\sin(u)\right\rangle \\\exp \left\langle u,u'\right\rangle &=\left\langle \exp u,u'\exp u\right\rangle \\\log \left\langle u,u'\right\rangle &=\left\langle \log(u),u'/u\right\rangle \quad (u>0)\\\left\langle u,u'\right\rangle ^{k}&=\left\langle u^{k},u'ku^{k-1}\right\rangle \quad (u\neq 0)\\\left|\left\langle u,u'\right\rangle \right|&=\left\langle \left|u\right|,u'\operatorname {sign} u\right\rangle \quad (u\neq 0)\end{aligned}}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e2eae6f532a78571f821dafd9e0b753c08b55818" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -17.005ex; width:45.215ex; height:35.176ex;" alt="{\displaystyle {\begin{aligned}\left\langle u,u'\right\rangle +\left\langle v,v'\right\rangle &=\left\langle u+v,u'+v'\right\rangle \\\left\langle u,u'\right\rangle -\left\langle v,v'\right\rangle &=\left\langle u-v,u'-v'\right\rangle \\\left\langle u,u'\right\rangle *\left\langle v,v'\right\rangle &=\left\langle uv,u'v+uv'\right\rangle \\\left\langle u,u'\right\rangle /\left\langle v,v'\right\rangle &=\left\langle {\frac {u}{v}},{\frac {u'v-uv'}{v^{2}}}\right\rangle \quad (v\neq 0)\\\sin \left\langle u,u'\right\rangle &=\left\langle \sin(u),u'\cos(u)\right\rangle \\\cos \left\langle u,u'\right\rangle &=\left\langle \cos(u),-u'\sin(u)\right\rangle \\\exp \left\langle u,u'\right\rangle &=\left\langle \exp u,u'\exp u\right\rangle \\\log \left\langle u,u'\right\rangle &=\left\langle \log(u),u'/u\right\rangle \quad (u>0)\\\left\langle u,u'\right\rangle ^{k}&=\left\langle u^{k},u'ku^{k-1}\right\rangle \quad (u\neq 0)\\\left|\left\langle u,u'\right\rangle \right|&=\left\langle \left|u\right|,u'\operatorname {sign} u\right\rangle \quad (u\neq 0)\end{aligned}}}" /> and in general for the primitive function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>g</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle g}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d3556280e66fe2c0d0140df20935a6f057381d77" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.116ex; height:2.009ex;" alt="{\displaystyle g}" />, <math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g(\langle u,u'\rangle ,\langle v,v'\rangle )=\langle g(u,v),g_{u}(u,v)u'+g_{v}(u,v)v'\rangle }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>g</mi> <mo stretchy="false">(</mo> <mo fence="false" stretchy="false">⟨</mo> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mo fence="false" stretchy="false">⟩</mo> <mo>,</mo> <mo fence="false" stretchy="false">⟨</mo> <mi>v</mi> <mo>,</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> <mo fence="false" stretchy="false">⟩</mo> <mo stretchy="false">)</mo> <mo>=</mo> <mo fence="false" stretchy="false">⟨</mo> <mi>g</mi> <mo stretchy="false">(</mo> <mi>u</mi> <mo>,</mo> <mi>v</mi> <mo stretchy="false">)</mo> <mo>,</mo> <msub> <mi>g</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>u</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>u</mi> <mo>,</mo> <mi>v</mi> <mo stretchy="false">)</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mo>+</mo> <msub> <mi>g</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>v</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>u</mi> <mo>,</mo> <mi>v</mi> <mo stretchy="false">)</mo> <msup> <mi>v</mi> <mo>′</mo> </msup> <mo fence="false" stretchy="false">⟩</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle g(\langle u,u'\rangle ,\langle v,v'\rangle )=\langle g(u,v),g_{u}(u,v)u'+g_{v}(u,v)v'\rangle }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bf6d40f99e4ed37d050c9d301c9a42ac589805ff" class="mwe-math-fallback-image-display mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:49.976ex; height:3.009ex;" alt="{\displaystyle g(\langle u,u'\rangle ,\langle v,v'\rangle )=\langle g(u,v),g_{u}(u,v)u'+g_{v}(u,v)v'\rangle }" /> where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g_{u}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>g</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>u</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle g_{u}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9b516e0dd7b63c2a37acd02c922c61aaea8350f9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.282ex; height:2.009ex;" alt="{\displaystyle g_{u}}" /> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g_{v}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>g</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>v</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle g_{v}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7bfb4a0665f0214680e9b9e12e179ac78fab0aba" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.139ex; height:2.009ex;" alt="{\displaystyle g_{v}}" /> are the derivatives of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>g</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle g}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d3556280e66fe2c0d0140df20935a6f057381d77" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.116ex; height:2.009ex;" alt="{\displaystyle g}" /> with respect to its first and second arguments, respectively. When a binary basic arithmetic operation is applied to mixed arguments—the pair <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \langle u,u'\rangle }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">⟨</mo> <mi>u</mi> <mo>,</mo> <msup> <mi>u</mi> <mo>′</mo> </msup> <mo fence="false" stretchy="false">⟩</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \langle u,u'\rangle }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/55128ea6b433f62c0b41b36c8291f371c220697c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.187ex; height:3.009ex;" alt="{\displaystyle \langle u,u'\rangle }" /> and the real number <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle c}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>c</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle c}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/86a67b81c2de995bd608d5b2df50cd8cd7d92455" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.007ex; height:1.676ex;" alt="{\displaystyle c}" />—the real number is first lifted to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \langle c,0\rangle }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">⟨</mo> <mi>c</mi> <mo>,</mo> <mn>0</mn> <mo fence="false" stretchy="false">⟩</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \langle c,0\rangle }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/39fd148c69c9dcd2e9276bbac3108834651b2e8d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.013ex; height:2.843ex;" alt="{\displaystyle \langle c,0\rangle }" />. The derivative of a function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f:\mathbb {R} \to \mathbb {R} }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>f</mi> <mo>:</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mo stretchy="false">→</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f:\mathbb {R} \to \mathbb {R} }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1e3a10a3ad05781f5cf9c2d875a02227e21a8448" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:10.186ex; height:2.509ex;" alt="{\displaystyle f:\mathbb {R} \to \mathbb {R} }" /> at the point <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{0}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{0}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/86f21d0e31751534cd6584264ecf864a6aa792cf" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.384ex; height:2.009ex;" alt="{\displaystyle x_{0}}" /> is now found by calculating <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f(\langle x_{0},1\rangle )}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>f</mi> <mo stretchy="false">(</mo> <mo fence="false" stretchy="false">⟨</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>,</mo> <mn>1</mn> <mo fence="false" stretchy="false">⟩</mo> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f(\langle x_{0},1\rangle )}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1c42af355140d275e97f4de15b1b8e3270ddcc23" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.477ex; height:2.843ex;" alt="{\displaystyle f(\langle x_{0},1\rangle )}" /> using the above arithmetic, which gives <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \langle f(x_{0}),f'(x_{0})\rangle }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">⟨</mo> <mi>f</mi> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>,</mo> <msup> <mi>f</mi> <mo>′</mo> </msup> <mo stretchy="false">(</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo fence="false" stretchy="false">⟩</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \langle f(x_{0}),f'(x_{0})\rangle }</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d67f9be8cbac4b614b67be0e0e232a0acc3aace0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.513ex; height:3.009ex;" alt="{\displaystyle \langle f(x_{0}),f'(x_{0})\rangle }" /> as the result. <div class="mw-heading mw-heading3"><h3 id="Implementation_3">Implementation</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=16" title="Edit section: Implementation">edit</a>]</div> An example implementation based on the dual number approach follows. <div class="mw-heading mw-heading4"><h4 id="Pseudo_code_2">Pseudo code</h4>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=17" title="Edit section: Pseudo code">edit</a>]</div> <style data-mw-deduplicate="TemplateStyles:r1195917819">.mw-parser-output .pre-borderless{border:none}</style><pre class="pre">Dual plus(Dual A, Dual B) { return { realPartOf(A) + realPartOf(B), infinitesimalPartOf(A) + infinitesimalPartOf(B) }; } Dual minus(Dual A, Dual B) { return { realPartOf(A) - realPartOf(B), infinitesimalPartOf(A) - infinitesimalPartOf(B) }; } Dual multiply(Dual A, Dual B) { return { realPartOf(A) * realPartOf(B), realPartOf(B) * infinitesimalPartOf(A) + realPartOf(A) * infinitesimalPartOf(B) }; } X = {x, 0}; Y = {y, 0}; Epsilon = {0, 1}; xPartial = infinitesimalPartOf(f(X + Epsilon, Y)); yPartial = infinitesimalPartOf(f(X, Y + Epsilon));</pre> <div class="mw-heading mw-heading4"><h4 id="C++_3">C++</h4>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=18" title="Edit section: C++">edit</a>]</div> <div class="mw-highlight mw-highlight-lang-cpp mw-content-ltr" dir="ltr"><pre>#include <iostream> struct Dual { float realPart, infinitesimalPart; Dual(float realPart, float infinitesimalPart=0): realPart(realPart), infinitesimalPart(infinitesimalPart) {} Dual operator+(Dual other) { return Dual( realPart + other.realPart, infinitesimalPart + other.infinitesimalPart ); } Dual operator*(Dual other) { return Dual( realPart * other.realPart, other.realPart * infinitesimalPart + realPart * other.infinitesimalPart ); } }; // Example: Finding the partials of z = x * (x + y) + y * y at (x, y) = (2, 3) Dual f(Dual x, Dual y) { return x * (x + y) + y * y; } int main () { Dual x = Dual(2); Dual y = Dual(3); Dual epsilon = Dual(0, 1); Dual a = f(x + epsilon, y); Dual b = f(x, y + epsilon); std::cout << "∂z/∂x = " << a.infinitesimalPart << ", " << "∂z/∂y = " << b.infinitesimalPart << std::endl; // Output: ∂z/∂x = 7, ∂z/∂y = 8 return 0; } </pre></div> <div class="mw-heading mw-heading3"><h3 id="Vector_arguments_and_functions">Vector arguments and functions</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=19" title="Edit section: Vector arguments and functions">edit</a>]</div> Multivariate functions can be handled with the same efficiency and mechanisms as univariate functions by adopting a directional derivative operator. That is, if it is sufficient to compute <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y'=\nabla f(x)\cdot x'}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>y</mi> <mo>′</mo> </msup> <mo>=</mo> <mi mathvariant="normal">∇</mi> <mi>f</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <msup> <mi>x</mi> <mo>′</mo> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y'=\nabla f(x)\cdot x'}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a44c656a6aea36e73e9f03d9b1ace4cdbf7cdbf7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.991ex; height:3.009ex;" alt="{\displaystyle y'=\nabla f(x)\cdot x'}" />, the directional derivative <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y'\in \mathbb {R} ^{m}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>y</mi> <mo>′</mo> </msup> <mo>∈</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>m</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle y'\in \mathbb {R} ^{m}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c67ef9202dff526962562f2c752c5083ae911f20" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.039ex; height:2.843ex;" alt="{\displaystyle y'\in \mathbb {R} ^{m}}" /> of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f:\mathbb {R} ^{n}\to \mathbb {R} ^{m}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>f</mi> <mo>:</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> <mo stretchy="false">→</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>m</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f:\mathbb {R} ^{n}\to \mathbb {R} ^{m}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/aad78382c3d23bcb4051b3148f1a23b1d0ba52e3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:13.079ex; height:2.676ex;" alt="{\displaystyle f:\mathbb {R} ^{n}\to \mathbb {R} ^{m}}" /> at <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x\in \mathbb {R} ^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>x</mi> <mo>∈</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x\in \mathbb {R} ^{n}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c520ee2cb6ccf8a93c89a8c58a8378796bd52e53" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.067ex; height:2.343ex;" alt="{\displaystyle x\in \mathbb {R} ^{n}}" /> in the direction <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x'\in \mathbb {R} ^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>x</mi> <mo>′</mo> </msup> <mo>∈</mo> <msup> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">R</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x'\in \mathbb {R} ^{n}}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/06aa39d86f089a0cdfd7a6c4afb309c10e612e20" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.752ex; height:2.509ex;" alt="{\displaystyle x'\in \mathbb {R} ^{n}}" /> may be calculated as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (\langle y_{1},y'_{1}\rangle ,\ldots ,\langle y_{m},y'_{m}\rangle )=f(\langle x_{1},x'_{1}\rangle ,\ldots ,\langle x_{n},x'_{n}\rangle )}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mo fence="false" stretchy="false">⟨</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> <mo>′</mo> </msubsup> <mo fence="false" stretchy="false">⟩</mo> <mo>,</mo> <mo>…</mo> <mo>,</mo> <mo fence="false" stretchy="false">⟨</mo> <msub> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>m</mi> </mrow> </msub> <mo>,</mo> <msubsup> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>m</mi> </mrow> <mo>′</mo> </msubsup> <mo fence="false" stretchy="false">⟩</mo> <mo stretchy="false">)</mo> <mo>=</mo> <mi>f</mi> <mo stretchy="false">(</mo> <mo fence="false" stretchy="false">⟨</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msubsup> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> <mo>′</mo> </msubsup> <mo fence="false" stretchy="false">⟩</mo> <mo>,</mo> <mo>…</mo> <mo>,</mo> <mo fence="false" stretchy="false">⟨</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo>,</mo> <msubsup> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> <mo>′</mo> </msubsup> <mo fence="false" stretchy="false">⟩</mo> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (\langle y_{1},y'_{1}\rangle ,\ldots ,\langle y_{m},y'_{m}\rangle )=f(\langle x_{1},x'_{1}\rangle ,\ldots ,\langle x_{n},x'_{n}\rangle )}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d2f204b7eb8c22152f3e5114f16d8dfba07bfe5c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:49.605ex; height:3.009ex;" alt="{\displaystyle (\langle y_{1},y'_{1}\rangle ,\ldots ,\langle y_{m},y'_{m}\rangle )=f(\langle x_{1},x'_{1}\rangle ,\ldots ,\langle x_{n},x'_{n}\rangle )}" /> using the same arithmetic as above. If all the elements of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \nabla f}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">∇</mi> <mi>f</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \nabla f}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b7b4d6de89b52c5a5e6e1583cb63eaee263e307b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.214ex; height:2.509ex;" alt="{\displaystyle \nabla f}" /> are desired, then <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>n</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a601995d55609f2d9f5e233e36fbe9ea26011b3b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}" /> function evaluations are required. Note that in many optimization applications, the directional derivative is indeed sufficient. <div class="mw-heading mw-heading3"><h3 id="High_order_and_many_variables">High order and many variables</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=20" title="Edit section: High order and many variables">edit</a>]</div> The above arithmetic can be generalized to calculate second order and higher derivatives of multivariate functions. However, the arithmetic rules quickly grow complicated: complexity is quadratic in the highest derivative degree. Instead, truncated <a href="/wiki/Taylor_series" title="Taylor series">Taylor polynomial</a> algebra can be used. The resulting arithmetic, defined on generalized dual numbers, allows efficient computation using functions as if they were a data type. Once the Taylor polynomial of a function is known, the derivatives are easily extracted. <div class="mw-heading mw-heading2"><h2 id="Implementation_4">Implementation</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=21" title="Edit section: Implementation">edit</a>]</div> Forward-mode AD is implemented by a <a href="/w/index.php?title=Nonstandard_interpretation&action=edit&redlink=1" class="new" title="Nonstandard interpretation (page does not exist)">nonstandard interpretation</a> of the program in which real numbers are replaced by dual numbers, constants are lifted to dual numbers with a zero epsilon coefficient, and the numeric primitives are lifted to operate on dual numbers. This nonstandard interpretation is generally implemented using one of two strategies: source code transformation or operator overloading. <div class="mw-heading mw-heading3"><h3 id="Source_code_transformation_(SCT)">Source code transformation (SCT)</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=22" title="Edit section: Source code transformation (SCT)">edit</a>]</div> <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:SourceTransformationAutomaticDifferentiation.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/8/80/SourceTransformationAutomaticDifferentiation.png/300px-SourceTransformationAutomaticDifferentiation.png" decoding="async" width="300" height="52" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/80/SourceTransformationAutomaticDifferentiation.png/450px-SourceTransformationAutomaticDifferentiation.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/80/SourceTransformationAutomaticDifferentiation.png/600px-SourceTransformationAutomaticDifferentiation.png 2x" data-file-width="2488" data-file-height="430" /></a><figcaption>Figure 4: Example of how source code transformation could work</figcaption></figure> The source code for a function is replaced by an automatically generated source code that includes statements for calculating the derivatives interleaved with the original instructions. <a href="/wiki/Source_code_transformation" class="mw-redirect" title="Source code transformation">Source code transformation</a> can be implemented for all programming languages, and it is also easier for the compiler to do compile time optimizations. However, the implementation of the AD tool itself is more difficult and the build system is more complex. <div class="mw-heading mw-heading3"><h3 id="Operator_overloading_(OO)">Operator overloading (OO)</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=23" title="Edit section: Operator overloading (OO)">edit</a>]</div> <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:OperatorOverloadingAutomaticDifferentiation.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/a/af/OperatorOverloadingAutomaticDifferentiation.png/330px-OperatorOverloadingAutomaticDifferentiation.png" decoding="async" width="300" height="74" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/a/af/OperatorOverloadingAutomaticDifferentiation.png/500px-OperatorOverloadingAutomaticDifferentiation.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/a/af/OperatorOverloadingAutomaticDifferentiation.png/960px-OperatorOverloadingAutomaticDifferentiation.png 2x" data-file-width="2498" data-file-height="620" /></a><figcaption>Figure 5: Example of how operator overloading could work</figcaption></figure> <a href="/wiki/Operator_overloading" title="Operator overloading">Operator overloading</a> is a possibility for source code written in a language supporting it. Objects for real numbers and elementary mathematical operations must be overloaded to cater for the augmented arithmetic depicted above. This requires no change in the form or sequence of operations in the original source code for the function to be differentiated, but often requires changes in basic data types for numbers and vectors to support overloading and often also involves the insertion of special flagging operations. Due to the inherent operator overloading overhead on each loop, this approach usually demonstrates weaker speed performance. <div class="mw-heading mw-heading3"><h3 id="Operator_overloading_and_source_code_transformation">Operator overloading and source code transformation</h3>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=24" title="Edit section: Operator overloading and source code transformation">edit</a>]</div> Overloaded Operators can be used to extract the valuation graph, followed by automatic generation of the AD-version of the primal function at run-time. Unlike the classic OO AAD, such AD-function does not change from one iteration to the next one. Hence there is any OO or tape interpretation run-time overhead per Xi sample. With the AD-function being generated at runtime, it can be optimised to take into account the current state of the program and precompute certain values. In addition, it can be generated in a way to consistently utilize native CPU vectorization to process 4(8)-double chunks of user data (AVX2\AVX512 speed up x4-x8). With multithreading added into account, such approach can lead to a final acceleration of order 8 × #Cores compared to the traditional AAD tools. A reference implementation is available on GitHub.<a href="#cite_note-22">[21]</a> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=25" title="Edit section: See also">edit</a>]</div> <ul><li><a href="/wiki/Differentiable_programming" title="Differentiable programming">Differentiable programming</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="Notes">Notes</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=26" title="Edit section: Notes">edit</a>]</div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist reflist-lower-alpha"> <div class="mw-references-wrap"><ol class="references"> <li id="cite_note-18"><a href="#cite_ref-18">^</a> In terms of weight matrices, the adjoint is the <a href="/wiki/Transpose" title="Transpose">transpose</a>. Addition is the <a href="/wiki/Covector" class="mw-redirect" title="Covector">covector</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle [1\cdots 1]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">[</mo> <mn>1</mn> <mo>⋯</mo> <mn>1</mn> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle [1\cdots 1]}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bf45494491a9127a08ac5f529d6c2e7fa9df6bd1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.116ex; height:2.843ex;" alt="{\displaystyle [1\cdots 1]}" />, since <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle [1\cdots 1]\left[{\begin{smallmatrix}x_{1}\\\vdots \\x_{n}\end{smallmatrix}}\right]=x_{1}+\cdots +x_{n},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">[</mo> <mn>1</mn> <mo>⋯</mo> <mn>1</mn> <mo stretchy="false">]</mo> <mrow> <mo>[</mo> <mrow class="MJX-TeXAtom-ORD"> <mstyle scriptlevel="1"> <mtable rowspacing=".2em" columnspacing="0.333em" displaystyle="false"> <mtr> <mtd> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd> <mo>⋮</mo> </mtd> </mtr> <mtr> <mtd> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mtd> </mtr> </mtable> </mstyle> </mrow> <mo>]</mo> </mrow> <mo>=</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>+</mo> <mo>⋯</mo> <mo>+</mo> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle [1\cdots 1]\left[{\begin{smallmatrix}x_{1}\\\vdots \\x_{n}\end{smallmatrix}}\right]=x_{1}+\cdots +x_{n},}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7a9a62ad6b33394bb459d2a5fd08b3ed4394cfe6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:29.696ex; height:6.176ex;" alt="{\displaystyle [1\cdots 1]\left[{\begin{smallmatrix}x_{1}\\\vdots \\x_{n}\end{smallmatrix}}\right]=x_{1}+\cdots +x_{n},}" /> and fanout is the vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \left[{\begin{smallmatrix}1\\\vdots \\1\end{smallmatrix}}\right],}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow> <mo>[</mo> <mrow class="MJX-TeXAtom-ORD"> <mstyle scriptlevel="1"> <mtable rowspacing=".2em" columnspacing="0.333em" displaystyle="false"> <mtr> <mtd> <mn>1</mn> </mtd> </mtr> <mtr> <mtd> <mo>⋮</mo> </mtd> </mtr> <mtr> <mtd> <mn>1</mn> </mtd> </mtr> </mtable> </mstyle> </mrow> <mo>]</mo> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \left[{\begin{smallmatrix}1\\\vdots \\1\end{smallmatrix}}\right],}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d41449e0a185da184e5a72cfc317e2d99cbf6116" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:5.062ex; height:6.176ex;" alt="{\displaystyle \left[{\begin{smallmatrix}1\\\vdots \\1\end{smallmatrix}}\right],}" /> since <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \left[{\begin{smallmatrix}1\\\vdots \\1\end{smallmatrix}}\right][x]=\left[{\begin{smallmatrix}x\\\vdots \\x\end{smallmatrix}}\right].}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow> <mo>[</mo> <mrow class="MJX-TeXAtom-ORD"> <mstyle scriptlevel="1"> <mtable rowspacing=".2em" columnspacing="0.333em" displaystyle="false"> <mtr> <mtd> <mn>1</mn> </mtd> </mtr> <mtr> <mtd> <mo>⋮</mo> </mtd> </mtr> <mtr> <mtd> <mn>1</mn> </mtd> </mtr> </mtable> </mstyle> </mrow> <mo>]</mo> </mrow> <mo stretchy="false">[</mo> <mi>x</mi> <mo stretchy="false">]</mo> <mo>=</mo> <mrow> <mo>[</mo> <mrow class="MJX-TeXAtom-ORD"> <mstyle scriptlevel="1"> <mtable rowspacing=".2em" columnspacing="0.333em" displaystyle="false"> <mtr> <mtd> <mi>x</mi> </mtd> </mtr> <mtr> <mtd> <mo>⋮</mo> </mtd> </mtr> <mtr> <mtd> <mi>x</mi> </mtd> </mtr> </mtable> </mstyle> </mrow> <mo>]</mo> </mrow> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \left[{\begin{smallmatrix}1\\\vdots \\1\end{smallmatrix}}\right][x]=\left[{\begin{smallmatrix}x\\\vdots \\x\end{smallmatrix}}\right].}</annotation> </semantics> </math><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d33d5fcc124ef2bdffd63c5c21a8ac5de293d50c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:15.318ex; height:6.176ex;" alt="{\displaystyle \left[{\begin{smallmatrix}1\\\vdots \\1\end{smallmatrix}}\right][x]=\left[{\begin{smallmatrix}x\\\vdots \\x\end{smallmatrix}}\right].}" /> </li> </ol></div></div> <div class="mw-heading mw-heading2"><h2 id="References">References</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=27" title="Edit section: References">edit</a>]</div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1239543626" /><div class="reflist reflist-columns references-column-width" style="column-width: 30em;"> <ol class="references"> <li id="cite_note-1"><a href="#cite_ref-1">^</a> <style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite id="CITEREFNeidinger2010" class="citation journal cs1">Neidinger, Richard D. (2010). <a rel="nofollow" class="external text" href="http://academics.davidson.edu/math/neidinger/SIAMRev74362.pdf">"Introduction to Automatic Differentiation and MATLAB Object-Oriented Programming"</a> (PDF). SIAM Review. 52 (3): 545–563. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.362.6580">10.1.1.362.6580</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1137%2F080743627">10.1137/080743627</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:17134969">17134969</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=SIAM+Review&rft.atitle=Introduction+to+Automatic+Differentiation+and+MATLAB+Object-Oriented+Programming&rft.volume=52&rft.issue=3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E545-%3C%2Fspan%3E563&rft.date=2010&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.362.6580%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A17134969%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1137%2F080743627&rft.aulast=Neidinger&rft.aufirst=Richard+D.&rft_id=http%3A%2F%2Facademics.davidson.edu%2Fmath%2Fneidinger%2FSIAMRev74362.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-baydin2018automatic-2">^ <a href="#cite_ref-baydin2018automatic_2-0">a</a> <a href="#cite_ref-baydin2018automatic_2-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBaydinPearlmutterRadulSiskind2018" class="citation journal cs1">Baydin, Atilim Gunes; Pearlmutter, Barak; Radul, Alexey Andreyevich; Siskind, Jeffrey (2018). <a rel="nofollow" class="external text" href="http://jmlr.org/papers/v18/17-468.html">"Automatic differentiation in machine learning: a survey"</a>. Journal of Machine Learning Research. 18: 1–43.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Machine+Learning+Research&rft.atitle=Automatic+differentiation+in+machine+learning%3A+a+survey&rft.volume=18&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1-%3C%2Fspan%3E43&rft.date=2018&rft.aulast=Baydin&rft.aufirst=Atilim+Gunes&rft.au=Pearlmutter%2C+Barak&rft.au=Radul%2C+Alexey+Andreyevich&rft.au=Siskind%2C+Jeffrey&rft_id=http%3A%2F%2Fjmlr.org%2Fpapers%2Fv18%2F17-468.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-Dawood.Megahed.2023-3">^ <a href="#cite_ref-Dawood.Megahed.2023_3-0">a</a> <a href="#cite_ref-Dawood.Megahed.2023_3-1">b</a> <a href="#cite_ref-Dawood.Megahed.2023_3-2">c</a> <a href="#cite_ref-Dawood.Megahed.2023_3-3">d</a> <a href="#cite_ref-Dawood.Megahed.2023_3-4">e</a> <a href="/w/index.php?title=Hend_Dawood&action=edit&redlink=1" class="new" title="Hend Dawood (page does not exist)">Hend Dawood</a> and <a href="/w/index.php?title=Nefertiti_Megahed&action=edit&redlink=1" class="new" title="Nefertiti Megahed (page does not exist)">Nefertiti Megahed</a> (2023). Automatic differentiation of uncertainties: an interval computational differentiation for first and higher derivatives with implementation. PeerJ Computer Science 9:e1301 <a rel="nofollow" class="external free" href="https://doi.org/10.7717/peerj-cs.1301">https://doi.org/10.7717/peerj-cs.1301</a>. </li> <li id="cite_note-Dawood.Megahed.2019-4">^ <a href="#cite_ref-Dawood.Megahed.2019_4-0">a</a> <a href="#cite_ref-Dawood.Megahed.2019_4-1">b</a> <a href="#cite_ref-Dawood.Megahed.2019_4-2">c</a> <a href="#cite_ref-Dawood.Megahed.2019_4-3">d</a> <a href="/w/index.php?title=Hend_Dawood&action=edit&redlink=1" class="new" title="Hend Dawood (page does not exist)">Hend Dawood</a> and <a href="/w/index.php?title=Nefertiti_Megahed&action=edit&redlink=1" class="new" title="Nefertiti Megahed (page does not exist)">Nefertiti Megahed</a> (2019). A Consistent and Categorical Axiomatization of Differentiation Arithmetic Applicable to First and Higher Order Derivatives. Punjab University Journal of Mathematics. 51(11). pp. 77-100. doi: 10.5281/zenodo.3479546. <a rel="nofollow" class="external free" href="https://doi.org/10.5281/zenodo.3479546">http://doi.org/10.5281/zenodo.3479546</a>. </li> <li id="cite_note-Dawood-attribution-5">^ <a href="#cite_ref-Dawood-attribution_5-0">a</a> <a href="#cite_ref-Dawood-attribution_5-1">b</a> <a href="//creativecommons.org/licenses/by-sa/4.0/" title="creativecommons:by-sa/4.0/"><img alt="" src="//upload.wikimedia.org/wikipedia/commons/thumb/d/d9/Creative_Commons_by-sa_small.svg/80px-Creative_Commons_by-sa_small.svg.png" decoding="async" width="80" height="15" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/d/d9/Creative_Commons_by-sa_small.svg/120px-Creative_Commons_by-sa_small.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/d/d9/Creative_Commons_by-sa_small.svg/160px-Creative_Commons_by-sa_small.svg.png 2x" data-file-width="80" data-file-height="15" /></a> This article incorporates <a rel="nofollow" class="external text" href="https://pmc.ncbi.nlm.nih.gov/articles/PMC10280627/">text</a> by Dawood and Megahed available under the <a href="//creativecommons.org/licenses/by-sa/4.0/" class="extiw" title="creativecommons:by-sa/4.0/">CC BY-SA 4.0</a> license. </li> <li id="cite_note-Dawood.Dawood.2022-6"><a href="#cite_ref-Dawood.Dawood.2022_6-0">^</a> <a href="/w/index.php?title=Hend_Dawood&action=edit&redlink=1" class="new" title="Hend Dawood (page does not exist)">Hend Dawood</a> and <a href="/w/index.php?title=Yasser_Dawood&action=edit&redlink=1" class="new" title="Yasser Dawood (page does not exist)">Yasser Dawood</a> (2022). Interval Root Finding and Interval Polynomials: Methods and Applications in Science and Engineering. In S. Chakraverty, editor, Polynomial Paradigms: Trends and Applications in Science and Engineering, chapter 15. IOP Publishing. ISBN 978-0-7503-5065-5. doi: 10.1088/978-0-7503-5067-9ch15. URL <a rel="nofollow" class="external free" href="https://doi.org/10.1088/978-0-7503-5067-9ch15">https://doi.org/10.1088/978-0-7503-5067-9ch15</a>. </li> <li id="cite_note-Rump.1999-7"><a href="#cite_ref-Rump.1999_7-0">^</a> <a href="/w/index.php?title=Siegfried_M._Rump&action=edit&redlink=1" class="new" title="Siegfried M. Rump (page does not exist)">Siegfried M. Rump</a> (1999). INTLAB–INTerval LABoratory. In T. Csendes, editor, Developments in Reliable Computing, pages 77–104. Kluwer Academic Publishers, Dordrecht. </li> <li id="cite_note-Chevillard.Joldes.Lauter.2010-8"><a href="#cite_ref-Chevillard.Joldes.Lauter.2010_8-0">^</a> S. Chevillard, M. Joldes, and C. Lauter. Sollya (2010). An Environment for the Development of Numerical Codes. In K. Fukuda, J. van der Hoeven, M. Joswig, and N. Takayama, editors, Mathematical Software - ICMS 2010, volume 6327 of Lecture Notes in Computer Science, pages 28–31, Heidelberg, Germany. Springer. </li> <li id="cite_note-Dawood.Inc4.2022-9"><a href="#cite_ref-Dawood.Inc4.2022_9-0">^</a> <a href="/w/index.php?title=Hend_Dawood&action=edit&redlink=1" class="new" title="Hend Dawood (page does not exist)">Hend Dawood</a> (2022). <a href="/w/index.php?title=InCLosure&action=edit&redlink=1" class="new" title="InCLosure (page does not exist)">InCLosure</a> (Interval enCLosure)–A Language and Environment for Reliable Scientific Computing. Computer Software, Version 4.0. Department of Mathematics. Faculty of Science, Cairo University, Giza, Egypt, September 2022. url: <a rel="nofollow" class="external free" href="https://doi.org/10.5281/zenodo.2702404">https://doi.org/10.5281/zenodo.2702404</a>. </li> <li id="cite_note-Fries.2019-10"><a href="#cite_ref-Fries.2019_10-0">^</a> Christian P. Fries (2019). Stochastic Automatic Differentiation: Automatic Differentiation for Monte-Carlo Simulations. Quantitative Finance, 19(6):1043–1059. doi: 10.1080/14697688.2018.1556398. url: <a rel="nofollow" class="external free" href="https://doi.org/10.1080/14697688.2018.1556398">https://doi.org/10.1080/14697688.2018.1556398</a>. </li> <li id="cite_note-Dawood.Dawood.2020-11"><a href="#cite_ref-Dawood.Dawood.2020_11-0">^</a> <a href="/w/index.php?title=Hend_Dawood&action=edit&redlink=1" class="new" title="Hend Dawood (page does not exist)">Hend Dawood</a> and <a href="/w/index.php?title=Yasser_Dawood&action=edit&redlink=1" class="new" title="Yasser Dawood (page does not exist)">Yasser Dawood</a> (2020). Universal Intervals: Towards a Dependency-Aware Interval Algebra. In S. Chakraverty, editor, Mathematical Methods in Interdisciplinary Sciences. chapter 10, pages 167–214. John Wiley & Sons, Hoboken, New Jersey. ISBN 978-1-119-58550-3. doi: 10.1002/9781119585640.ch10. url: <a rel="nofollow" class="external free" href="https://doi.org/10.1002/9781119585640.ch10">https://doi.org/10.1002/9781119585640.ch10</a>. </li> <li id="cite_note-Dawood.2014-12"><a href="#cite_ref-Dawood.2014_12-0">^</a> <a href="/w/index.php?title=Hend_Dawood&action=edit&redlink=1" class="new" title="Hend Dawood (page does not exist)">Hend Dawood</a> (2014). Interval Mathematics as a Potential Weapon against Uncertainty. In S. Chakraverty, editor, Mathematics of Uncertainty Modeling in the Analysis of Engineering and Science Problems. chapter 1, pages 1–38. IGI Global, Hershey, PA. ISBN 978-1-4666-4991-0. </li> <li id="cite_note-Wengert1964-13">^ <a href="#cite_ref-Wengert1964_13-0">a</a> <a href="#cite_ref-Wengert1964_13-1">b</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFR.E._Wengert1964" class="citation journal cs1">R.E. Wengert (1964). <a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F355586.364791">"A simple automatic derivative evaluation program"</a>. Comm. ACM. 7 (8): 463–464. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F355586.364791">10.1145/355586.364791</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:24039274">24039274</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Comm.+ACM&rft.atitle=A+simple+automatic+derivative+evaluation+program&rft.volume=7&rft.issue=8&rft.pages=%3Cspan+class%3D%22nowrap%22%3E463-%3C%2Fspan%3E464&rft.date=1964&rft_id=info%3Adoi%2F10.1145%2F355586.364791&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A24039274%23id-name%3DS2CID&rft.au=R.E.+Wengert&rft_id=https%3A%2F%2Fdoi.org%2F10.1145%252F355586.364791&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-grie2012-14"><a href="#cite_ref-grie2012_14-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGriewank2012" class="citation book cs1">Griewank, Andreas (2012). <a rel="nofollow" class="external text" href="https://ftp.gwdg.de/pub/misc/EMIS/journals/DMJDMV/vol-ismp/52_griewank-andreas-b.pdf">"Who invented the reverse mode of differentiation?"</a> (PDF). Optimization Stories. Documenta Mathematica Series. Vol. 6. pp. 389–400. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.4171%2Fdms%2F6%2F38">10.4171/dms/6/38</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-936609-58-5" title="Special:BookSources/978-3-936609-58-5"><bdi>978-3-936609-58-5</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Who+invented+the+reverse+mode+of+differentiation%3F&rft.btitle=Optimization+Stories&rft.series=Documenta+Mathematica+Series&rft.pages=%3Cspan+class%3D%22nowrap%22%3E389-%3C%2Fspan%3E400&rft.date=2012&rft_id=info%3Adoi%2F10.4171%2Fdms%2F6%2F38&rft.isbn=978-3-936609-58-5&rft.aulast=Griewank&rft.aufirst=Andreas&rft_id=https%3A%2F%2Fftp.gwdg.de%2Fpub%2Fmisc%2FEMIS%2Fjournals%2FDMJDMV%2Fvol-ismp%2F52_griewank-andreas-b.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-lin1976-15"><a href="#cite_ref-lin1976_15-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFLinnainmaa1976" class="citation journal cs1">Linnainmaa, Seppo (1976). "Taylor Expansion of the Accumulated Rounding Error". BIT Numerical Mathematics. 16 (2): 146–160. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2FBF01931367">10.1007/BF01931367</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:122357351">122357351</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=BIT+Numerical+Mathematics&rft.atitle=Taylor+Expansion+of+the+Accumulated+Rounding+Error&rft.volume=16&rft.issue=2&rft.pages=%3Cspan+class%3D%22nowrap%22%3E146-%3C%2Fspan%3E160&rft.date=1976&rft_id=info%3Adoi%2F10.1007%2FBF01931367&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A122357351%23id-name%3DS2CID&rft.aulast=Linnainmaa&rft.aufirst=Seppo&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-demm22-16"><a href="#cite_ref-demm22_16-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFMaximilian_E._Schüle,_Maximilian_Springer,_Alfons_Kemper,_Thomas_Neumann2022" class="citation book cs1">Maximilian E. Schüle, Maximilian Springer, <a href="/wiki/Alfons_Kemper" title="Alfons Kemper">Alfons Kemper</a>, <a href="/wiki/Thomas_Neumann" title="Thomas Neumann">Thomas Neumann</a> (2022). "LLVM code optimisation for automatic differentiation". Proceedings of the Sixth Workshop on Data Management for End-To-End Machine Learning. pp. 1–4. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F3533028.3533302">10.1145/3533028.3533302</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781450393751" title="Special:BookSources/9781450393751"><bdi>9781450393751</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:248853034">248853034</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=LLVM+code+optimisation+for+automatic+differentiation&rft.btitle=Proceedings+of+the+Sixth+Workshop+on+Data+Management+for+End-To-End+Machine+Learning&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1-%3C%2Fspan%3E4&rft.date=2022&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A248853034%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F3533028.3533302&rft.isbn=9781450393751&rft.au=Maximilian+E.+Sch%C3%BCle%2C+Maximilian+Springer%2C+Alfons+Kemper%2C+Thomas+Neumann&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"><code class="cs1-code">{{<a href="/wiki/Template:Cite_book" title="Template:Cite book">cite book</a>}}</code>: CS1 maint: multiple names: authors list (<a href="/wiki/Category:CS1_maint:_multiple_names:_authors_list" title="Category:CS1 maint: multiple names: authors list">link</a>) </li> <li id="cite_note-17"><a href="#cite_ref-17">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFBartholomew-BiggsBrownChristiansonDixon2000" class="citation journal cs1">Bartholomew-Biggs, Michael; Brown, Steven; Christianson, Bruce; Dixon, Laurence (2000). "Automatic differentiation of algorithms". Journal of Computational and Applied Mathematics. 124 (1–2): 171–190. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2000JCoAM.124..171B">2000JCoAM.124..171B</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0377-0427%2800%2900422-2">10.1016/S0377-0427(00)00422-2</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/2299%2F3010">2299/3010</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Computational+and+Applied+Mathematics&rft.atitle=Automatic+differentiation+of+algorithms&rft.volume=124&rft.issue=%3Cspan+class%3D%22nowrap%22%3E1%E2%80%93%3C%2Fspan%3E2&rft.pages=%3Cspan+class%3D%22nowrap%22%3E171-%3C%2Fspan%3E190&rft.date=2000&rft_id=info%3Ahdl%2F2299%2F3010&rft_id=info%3Adoi%2F10.1016%2FS0377-0427%2800%2900422-2&rft_id=info%3Abibcode%2F2000JCoAM.124..171B&rft.aulast=Bartholomew-Biggs&rft.aufirst=Michael&rft.au=Brown%2C+Steven&rft.au=Christianson%2C+Bruce&rft.au=Dixon%2C+Laurence&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-ssdbm21-19"><a href="#cite_ref-ssdbm21_19-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFMaximilian_E._Schüle,_Harald_Lang,_Maximilian_Springer,_Alfons_Kemper,_Thomas_Neumann,_Stephan_Günnemann2021" class="citation book cs1">Maximilian E. Schüle, Harald Lang, Maximilian Springer, <a href="/wiki/Alfons_Kemper" title="Alfons Kemper">Alfons Kemper</a>, <a href="/wiki/Thomas_Neumann" title="Thomas Neumann">Thomas Neumann</a>, Stephan Günnemann (2021). "In-Database Machine Learning with SQL on GPUs". 33rd International Conference on Scientific and Statistical Database Management. pp. 25–36. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F3468791.3468840">10.1145/3468791.3468840</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781450384131" title="Special:BookSources/9781450384131"><bdi>9781450384131</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:235386969">235386969</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=In-Database+Machine+Learning+with+SQL+on+GPUs&rft.btitle=33rd+International+Conference+on+Scientific+and+Statistical+Database+Management&rft.pages=%3Cspan+class%3D%22nowrap%22%3E25-%3C%2Fspan%3E36&rft.date=2021&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A235386969%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F3468791.3468840&rft.isbn=9781450384131&rft.au=Maximilian+E.+Sch%C3%BCle%2C+Harald+Lang%2C+Maximilian+Springer%2C+Alfons+Kemper%2C+Thomas+Neumann%2C+Stephan+G%C3%BCnnemann&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"><code class="cs1-code">{{<a href="/wiki/Template:Cite_book" title="Template:Cite book">cite book</a>}}</code>: CS1 maint: multiple names: authors list (<a href="/wiki/Category:CS1_maint:_multiple_names:_authors_list" title="Category:CS1 maint: multiple names: authors list">link</a>) </li> <li id="cite_note-dpd-20"><a href="#cite_ref-dpd_20-0">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFMaximilian_E._Schüle,_Harald_Lang,_Maximilian_Springer,_Alfons_Kemper,_Thomas_Neumann,_Stephan_Günnemann2022" class="citation journal cs1">Maximilian E. Schüle, Harald Lang, Maximilian Springer, <a href="/wiki/Alfons_Kemper" title="Alfons Kemper">Alfons Kemper</a>, <a href="/wiki/Thomas_Neumann" title="Thomas Neumann">Thomas Neumann</a>, Stephan Günnemann (2022). <a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10619-022-07417-7">"Recursive SQL and GPU-support for in-database machine learning"</a>. Distributed and Parallel Databases. 40 (2–3): 205–259. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10619-022-07417-7">10.1007/s10619-022-07417-7</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:250412395">250412395</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Distributed+and+Parallel+Databases&rft.atitle=Recursive+SQL+and+GPU-support+for+in-database+machine+learning&rft.volume=40&rft.issue=%3Cspan+class%3D%22nowrap%22%3E2%E2%80%93%3C%2Fspan%3E3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E205-%3C%2Fspan%3E259&rft.date=2022&rft_id=info%3Adoi%2F10.1007%2Fs10619-022-07417-7&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A250412395%23id-name%3DS2CID&rft.au=Maximilian+E.+Sch%C3%BCle%2C+Harald+Lang%2C+Maximilian+Springer%2C+Alfons+Kemper%2C+Thomas+Neumann%2C+Stephan+G%C3%BCnnemann&rft_id=https%3A%2F%2Fdoi.org%2F10.1007%252Fs10619-022-07417-7&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"><code class="cs1-code">{{<a href="/wiki/Template:Cite_journal" title="Template:Cite journal">cite journal</a>}}</code>: CS1 maint: multiple names: authors list (<a href="/wiki/Category:CS1_maint:_multiple_names:_authors_list" title="Category:CS1 maint: multiple names: authors list">link</a>) </li> <li id="cite_note-21"><a href="#cite_ref-21">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFNaumann2008" class="citation journal cs1">Naumann, Uwe (April 2008). "Optimal Jacobian accumulation is NP-complete". Mathematical Programming. 112 (2): 427–441. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.320.5665">10.1.1.320.5665</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10107-006-0042-z">10.1007/s10107-006-0042-z</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:30219572">30219572</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Mathematical+Programming&rft.atitle=Optimal+Jacobian+accumulation+is+NP-complete&rft.volume=112&rft.issue=2&rft.pages=%3Cspan+class%3D%22nowrap%22%3E427-%3C%2Fspan%3E441&rft.date=2008-04&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.320.5665%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A30219572%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1007%2Fs10107-006-0042-z&rft.aulast=Naumann&rft.aufirst=Uwe&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> <li id="cite_note-22"><a href="#cite_ref-22">^</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://github.com/matlogica/aadc-prototype">"AADC Prototype Library"</a>. June 22, 2022 – via GitHub.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=AADC+Prototype+Library&rft.date=2022-06-22&rft_id=https%3A%2F%2Fgithub.com%2Fmatlogica%2Faadc-prototype&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"> </li> </ol></div> <div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=28" title="Edit section: Further reading">edit</a>]</div> <ul><li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFRall1981" class="citation book cs1">Rall, Louis B. (1981). Automatic Differentiation: Techniques and Applications. Lecture Notes in Computer Science. Vol. 120. <a href="/wiki/Springer_Science%2BBusiness_Media" title="Springer Science+Business Media">Springer</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-10861-0" title="Special:BookSources/978-3-540-10861-0"><bdi>978-3-540-10861-0</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Automatic+Differentiation%3A+Techniques+and+Applications&rft.series=Lecture+Notes+in+Computer+Science&rft.pub=Springer&rft.date=1981&rft.isbn=978-3-540-10861-0&rft.aulast=Rall&rft.aufirst=Louis+B.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFGriewankWalther2008" class="citation book cs1">Griewank, Andreas; <a href="/wiki/Andrea_Walther" title="Andrea Walther">Walther, Andrea</a> (2008). <a rel="nofollow" class="external text" href="https://epubs.siam.org/doi/book/10.1137/1.9780898717761">Evaluating Derivatives: Principles and Techniques of Algorithmic Differentiation</a>. Other Titles in Applied Mathematics. Vol. 105 (2nd ed.). <a href="/wiki/Society_for_Industrial_and_Applied_Mathematics" title="Society for Industrial and Applied Mathematics">SIAM</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1137%2F1.9780898717761">10.1137/1.9780898717761</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-89871-659-7" title="Special:BookSources/978-0-89871-659-7"><bdi>978-0-89871-659-7</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Evaluating+Derivatives%3A+Principles+and+Techniques+of+Algorithmic+Differentiation&rft.series=Other+Titles+in+Applied+Mathematics&rft.edition=2nd&rft.pub=SIAM&rft.date=2008&rft_id=info%3Adoi%2F10.1137%2F1.9780898717761&rft.isbn=978-0-89871-659-7&rft.aulast=Griewank&rft.aufirst=Andreas&rft.au=Walther%2C+Andrea&rft_id=https%3A%2F%2Fepubs.siam.org%2Fdoi%2Fbook%2F10.1137%2F1.9780898717761&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFNeidinger2010" class="citation journal cs1">Neidinger, Richard (2010). <a rel="nofollow" class="external text" href="http://academics.davidson.edu/math/neidinger/SIAMRev74362.pdf">"Introduction to Automatic Differentiation and MATLAB Object-Oriented Programming"</a> (PDF). SIAM Review. 52 (3): 545–563. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.362.6580">10.1.1.362.6580</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1137%2F080743627">10.1137/080743627</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:17134969">17134969</a>. Retrieved 2013-03-15.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=SIAM+Review&rft.atitle=Introduction+to+Automatic+Differentiation+and+MATLAB+Object-Oriented+Programming&rft.volume=52&rft.issue=3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E545-%3C%2Fspan%3E563&rft.date=2010&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.362.6580%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A17134969%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1137%2F080743627&rft.aulast=Neidinger&rft.aufirst=Richard&rft_id=http%3A%2F%2Facademics.davidson.edu%2Fmath%2Fneidinger%2FSIAMRev74362.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFNaumann2012" class="citation book cs1">Naumann, Uwe (2012). The Art of Differentiating Computer Programs. Software-Environments-tools. <a href="/wiki/Society_for_Industrial_and_Applied_Mathematics" title="Society for Industrial and Applied Mathematics">SIAM</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-611972-06-1" title="Special:BookSources/978-1-611972-06-1"><bdi>978-1-611972-06-1</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=The+Art+of+Differentiating+Computer+Programs&rft.series=Software-Environments-tools&rft.pub=SIAM&rft.date=2012&rft.isbn=978-1-611972-06-1&rft.aulast=Naumann&rft.aufirst=Uwe&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222" /><cite id="CITEREFHenrard2017" class="citation book cs1">Henrard, Marc (2017). Algorithmic Differentiation in Finance Explained. Financial Engineering Explained. <a href="/wiki/Palgrave_Macmillan" title="Palgrave Macmillan">Palgrave Macmillan</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-319-53978-2" title="Special:BookSources/978-3-319-53978-2"><bdi>978-3-319-53978-2</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Algorithmic+Differentiation+in+Finance+Explained&rft.series=Financial+Engineering+Explained&rft.pub=Palgrave+Macmillan&rft.date=2017&rft.isbn=978-3-319-53978-2&rft.aulast=Henrard&rft.aufirst=Marc&rfr_id=info%3Asid%2Fen.wikipedia.org%3AAutomatic+differentiation" class="Z3988"></li></ul> <div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2>[<a href="/w/index.php?title=Automatic_differentiation&action=edit&section=29" title="Edit section: External links">edit</a>]</div> <ul><li><a rel="nofollow" class="external text" href="http://www.autodiff.org/">www.autodiff.org</a>, An "entry site to everything you want to know about automatic differentiation"</li> <li><a rel="nofollow" class="external text" href="http://www.autodiff.org/?module=Applications&application=HC1">Automatic Differentiation of Parallel OpenMP Programs</a></li> <li><a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/241730000_Automatic_Differentiation_C_Templates_and_Photogrammetry">Automatic Differentiation, C++ Templates and Photogrammetry</a></li> <li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20070927120356/http://www.vivlabs.com/subpage_ad.php">Automatic Differentiation, Operator Overloading Approach</a></li> <li><a rel="nofollow" class="external text" href="http://tapenade.inria.fr:8080/tapenade/index.jsp">Compute analytic derivatives of any Fortran77, Fortran95, or C program through a web-based interface</a> Automatic Differentiation of Fortran programs</li> <li><a rel="nofollow" class="external text" href="http://www.win-vector.com/dfiles/AutomaticDifferentiationWithScala.pdf">Description and example code for forward Automatic Differentiation in Scala</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20160803214549/http://www.win-vector.com/dfiles/AutomaticDifferentiationWithScala.pdf">Archived</a> 2016-08-03 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a></li> <li><a rel="nofollow" class="external text" href="https://www.finmath.net/finmath-lib/concepts/stochasticautomaticdifferentiation/">finmath-lib stochastic automatic differentiation</a>, Automatic differentiation for random variables (Java implementation of the stochastic automatic differentiation).</li> <li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20140423121504/http://developers.opengamma.com/quantitative-research/Adjoint-Algorithmic-Differentiation-OpenGamma.pdf">Adjoint Algorithmic Differentiation: Calibration and Implicit Function Theorem</a></li> <li><a rel="nofollow" class="external text" href="http://www.quantandfinancial.com/2017/02/automatic-differentiation-templated.html">C++ Template-based automatic differentiation article</a> and <a rel="nofollow" class="external text" href="https://github.com/omartinsky/QuantAndFinancial/tree/master/autodiff_templated">implementation</a></li> <li><a rel="nofollow" class="external text" href="https://github.com/google/tangent">Tangent</a> <a rel="nofollow" class="external text" href="https://research.googleblog.com/2017/11/tangent-source-to-source-debuggable.html">Source-to-Source Debuggable Derivatives</a></li> <li><a rel="nofollow" class="external text" href="http://www.nag.co.uk/doc/techrep/pdf/tr5_10.pdf">Exact First- and Second-Order Greeks by Algorithmic Differentiation</a></li> <li><a rel="nofollow" class="external text" href="http://www.nag.co.uk/Market/articles/adjoint-algorithmic-differentiation-of-gpu-accelerated-app.pdf">Adjoint Algorithmic Differentiation of a GPU Accelerated Application</a></li> <li><a rel="nofollow" class="external text" href="http://www.nag.co.uk/Market/seminars/Uwe_AD_Slides_July13.pdf">Adjoint Methods in Computational Finance Software Tool Support for Algorithmic Differentiationop</a></li> <li><a rel="nofollow" class="external text" href="https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/xva-pricing-application-financial-services-white-papers.pdf">More than a Thousand Fold Speed Up for xVA Pricing Calculations with Intel Xeon Scalable Processors</a></li> <li><a rel="nofollow" class="external text" href="https://github.com/ExcessPhase/ctaylor">Sparse truncated Taylor series implementation with VBIC95 example for higher order derivatives</a></li></ul> <div class="navbox-styles"><style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="Differentiable_computing254" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Differentiable_computing" title="Template:Differentiable computing"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Differentiable_computing" title="Template talk:Differentiable computing"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Differentiable_computing" title="Special:EditPage/Template:Differentiable computing"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Differentiable_computing254" style="font-size:114%;margin:0 4em">Differentiable computing</div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Differentiable_function" title="Differentiable function">General</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Differentiable_programming" title="Differentiable programming">Differentiable programming</a></li> <li><a href="/wiki/Information_geometry" title="Information geometry">Information geometry</a></li> <li><a href="/wiki/Statistical_manifold" title="Statistical manifold">Statistical manifold</a></li> <li><a class="mw-selflink selflink">Automatic differentiation</a></li> <li><a href="/wiki/Neuromorphic_computing" title="Neuromorphic computing">Neuromorphic computing</a></li> <li><a href="/wiki/Pattern_recognition" title="Pattern recognition">Pattern recognition</a></li> <li><a href="/wiki/Ricci_calculus" title="Ricci calculus">Ricci calculus</a></li> <li><a href="/wiki/Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li> <li><a href="/wiki/Inductive_bias" title="Inductive bias">Inductive bias</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Hardware</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Graphcore" title="Graphcore">IPU</a></li> <li><a href="/wiki/Tensor_Processing_Unit" title="Tensor Processing Unit">TPU</a></li> <li><a href="/wiki/Vision_processing_unit" title="Vision processing unit">VPU</a></li> <li><a href="/wiki/Memristor" title="Memristor">Memristor</a></li> <li><a href="/wiki/SpiNNaker" title="SpiNNaker">SpiNNaker</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Software libraries</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/TensorFlow" title="TensorFlow">TensorFlow</a></li> <li><a href="/wiki/PyTorch" title="PyTorch">PyTorch</a></li> <li><a href="/wiki/Keras" title="Keras">Keras</a></li> <li><a href="/wiki/Scikit-learn" title="Scikit-learn">scikit-learn</a></li> <li><a href="/wiki/Theano_(software)" title="Theano (software)">Theano</a></li> <li><a href="/wiki/JAX_(software)" title="JAX (software)">JAX</a></li> <li><a href="/wiki/Flux_(machine-learning_framework)" title="Flux (machine-learning framework)">Flux.jl</a></li> <li><a href="/wiki/MindSpore" title="MindSpore">MindSpore</a></li></ul> </div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><a href="/wiki/File:Symbol_portal_class.svg" class="mw-file-description" title="Portal"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/20px-Symbol_portal_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/e/e2/Symbol_portal_class.svg/40px-Symbol_portal_class.svg.png 1.5x" data-file-width="180" data-file-height="185" /></a> Portals <ul><li><a href="/wiki/Portal:Computer_programming" title="Portal:Computer programming">Computer programming</a></li> <li><a href="/wiki/Portal:Technology" title="Portal:Technology">Technology</a></li></ul></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374" /><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235" /></div><div role="navigation" class="navbox authority-control" aria-label="Navbox507" style="padding:3px"><table class="nowraplinks hlist navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Help:Authority_control" title="Help:Authority control">Authority control databases</a>: National <a href="https://www.wikidata.org/wiki/Q787371#identifiers" title="Edit this at Wikidata"><img alt="Edit this at Wikidata" src="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/10px-OOjs_UI_icon_edit-ltr-progressive.svg.png" decoding="async" width="10" height="10" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/15px-OOjs_UI_icon_edit-ltr-progressive.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/20px-OOjs_UI_icon_edit-ltr-progressive.svg.png 2x" data-file-width="20" data-file-height="20" /></a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"><ul><li><a rel="nofollow" class="external text" href="https://id.loc.gov/authorities/sh2011004690">United States</a></li><li><a rel="nofollow" class="external text" href="https://www.nli.org.il/en/authorities/987012430911405171">Israel</a></li></ul></div></td></tr></tbody></table></div>    </div><noscript><img src="https://auth.wikimedia.org/loginwiki/wiki/Special:CentralAutoLogin/start?useformat=desktop&type=1x1&usesul3=1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Automatic_differentiation&oldid=1283271943">https://en.wikipedia.org/w/index.php?title=Automatic_differentiation&oldid=1283271943</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Differential_calculus" title="Category:Differential calculus">Differential calculus</a></li><li><a href="/wiki/Category:Computer_algebra" title="Category:Computer algebra">Computer algebra</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:Articles_with_imported_Creative_Commons_Attribution-ShareAlike_4.0_text" title="Category:Articles with imported Creative Commons Attribution-ShareAlike 4.0 text">Articles with imported Creative Commons Attribution-ShareAlike 4.0 text</a></li><li><a href="/wiki/Category:CS1_maint:_multiple_names:_authors_list" title="Category:CS1 maint: multiple names: authors list">CS1 maint: multiple names: authors list</a></li><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_is_different_from_Wikidata" title="Category:Short description is different from Wikidata">Short description is different from Wikidata</a></li><li><a href="/wiki/Category:Webarchive_template_wayback_links" title="Category:Webarchive template wayback links">Webarchive template wayback links</a></li><li><a href="/wiki/Category:Articles_with_example_pseudocode" title="Category:Articles with example pseudocode">Articles with example pseudocode</a></li><li><a href="/wiki/Category:Articles_with_example_Python_(programming_language)_code" title="Category:Articles with example Python (programming language) code">Articles with example Python (programming language) code</a></li><li><a href="/wiki/Category:Articles_with_example_C%2B%2B_code" title="Category:Articles with example C++ code">Articles with example C++ code</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 31 March 2025, at 13:31 (UTC).</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Automatic_differentiation&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://www.wikimedia.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><picture><source media="(min-width: 500px)" srcset="/static/images/footer/wikimedia-button.svg" width="84" height="29"><img src="/static/images/footer/wikimedia.svg" width="25" height="25" alt="Wikimedia Foundation" lang="en" loading="lazy"></picture></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><picture><source media="(min-width: 500px)" srcset="/w/resources/assets/poweredby_mediawiki.svg" width="88" height="31"><img src="/w/resources/assets/mediawiki_compact.svg" alt="Powered by MediaWiki" lang="en" width="25" height="25" loading="lazy"></picture></a></li> </ul> </footer> </div> </div> </div> <div class="vector-header-container vector-sticky-header-container"> <div id="vector-sticky-header" class="vector-sticky-header"> <div class="vector-sticky-header-start"> <div class="vector-sticky-header-icon-start vector-button-flush-left vector-button-flush-right" aria-hidden="true"> <button class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-sticky-header-search-toggle" tabindex="-1" data-event-name="ui.vector-sticky-search-form.icon"> Search </button> </div> <div role="search" class="vector-search-box-vue vector-search-box-show-thumbnail vector-search-box"> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail"> <form action="/w/index.php" id="vector-sticky-search-form" class="cdx-search-input cdx-search-input--has-end-button"> <div class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia"> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <div class="vector-sticky-header-context-bar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-sticky-header-toc" class="vector-dropdown mw-portlet mw-portlet-sticky-header-toc vector-sticky-header-toc vector-button-flush-left" > <input type="checkbox" id="vector-sticky-header-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-sticky-header-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-sticky-header-toc-label" for="vector-sticky-header-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" > Toggle the table of contents </label> <div class="vector-dropdown-content"> <div id="vector-sticky-header-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div class="vector-sticky-header-context-bar-primary" aria-hidden="true" >Automatic differentiation</div> </div> </div> <div class="vector-sticky-header-end" aria-hidden="true"> <div class="vector-sticky-header-icons"> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-talk-sticky-header" tabindex="-1" data-event-name="talk-sticky-header"> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-subject-sticky-header" tabindex="-1" data-event-name="subject-sticky-header"> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-history-sticky-header" tabindex="-1" data-event-name="history-sticky-header"> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only mw-watchlink" id="ca-watchstar-sticky-header" tabindex="-1" data-event-name="watch-sticky-header"> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-edit-sticky-header" tabindex="-1" data-event-name="wikitext-edit-sticky-header"> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-ve-edit-sticky-header" tabindex="-1" data-event-name="ve-edit-sticky-header"> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-viewsource-sticky-header" tabindex="-1" data-event-name="ve-edit-protected-sticky-header"> </a> </div> <div class="vector-sticky-header-buttons"> <button class="cdx-button cdx-button--weight-quiet mw-interlanguage-selector" id="p-lang-btn-sticky-header" tabindex="-1" data-event-name="ui.dropdown-p-lang-btn-sticky-header"> 11 languages </button> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive" id="ca-addsection-sticky-header" tabindex="-1" data-event-name="addsection-sticky-header"> Add topic </a> </div> <div class="vector-sticky-header-icon-end"> <div class="vector-user-links"> </div> </div> </div> </div> </div> <div class="mw-portlet mw-portlet-dock-bottom emptyPortlet" id="p-dock-bottom"> <ul> </ul> </div> <script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-65bcbb4cf5-kwdpx","wgBackendResponseTime":260,"wgPageParseReport":{"limitreport":{"cputime":"0.473","walltime":"0.733","ppvisitednodes":{"value":2791,"limit":1000000},"postexpandincludesize":{"value":57042,"limit":2097152},"templateargumentsize":{"value":3998,"limit":2097152},"expansiondepth":{"value":11,"limit":100},"expensivefunctioncount":{"value":6,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":129721,"limit":5000000},"entityaccesscount":{"value":1,"limit":400},"timingprofile":["100.00% 374.912 1 -total"," 35.57% 133.346 2 Template:Reflist"," 24.59% 92.191 8 Template:Cite_journal"," 23.01% 86.263 1 Template:Short_description"," 17.87% 67.004 1 Template:Differentiable_computing"," 17.26% 64.702 1 Template:Navbox"," 13.66% 51.229 2 Template:Pagetype"," 7.68% 28.811 21 Template:Main_other"," 7.64% 28.635 7 Template:Cite_book"," 5.70% 21.366 1 Template:Authority_control"]},"scribunto":{"limitreport-timeusage":{"value":"0.219","limit":"10.000"},"limitreport-memusage":{"value":5046844,"limit":52428800}},"cachereport":{"origin":"mw-web.codfw.main-68bd6b7bf8-wrzl6","timestamp":"20250331133201","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Automatic differentiation","url":"https:\/\/en.wikipedia.org\/wiki\/Automatic_differentiation","sameAs":"http:\/\/www.wikidata.org\/entity\/Q787371","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q787371","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2004-06-18T12:47:41Z","dateModified":"2025-03-31T13:31:56Z","headline":"set of computer programming techniques to speedily compute derivatives"}</script> </body> </html>

CINXE.COM

Automatic differentiation - Wikipedia