Mutual information

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Mutual information - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"bb3a5ef0-5f8f-4a98-acda-1c7da808df33","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Mutual_information","wgTitle":"Mutual information","wgCurRevisionId":1270339045,"wgRevisionId":1270339045,"wgArticleId":427282,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["CS1: unfit URL","Articles with short description","Short description matches Wikidata","Information theory","Entropy and information"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Mutual_information","wgRelevantArticleId":427282,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false, "wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":60000,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q252973","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles" :"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.math.styles":"ready","ext.cite.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","mediawiki.page.media","ext.scribunto.logs","site","mediawiki.page.ready","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns", "ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cext.wikimediamessages.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.16"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Entropy-mutual-information-relative-entropy-relation-diagram.svg/1200px-Entropy-mutual-information-relative-entropy-relation-diagram.svg.png"> <meta property="og:image:width" content="1200"> <meta property="og:image:height" content="845"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Entropy-mutual-information-relative-entropy-relation-diagram.svg/800px-Entropy-mutual-information-relative-entropy-relation-diagram.svg.png"> <meta property="og:image:width" content="800"> <meta property="og:image:height" content="563"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Entropy-mutual-information-relative-entropy-relation-diagram.svg/640px-Entropy-mutual-information-relative-entropy-relation-diagram.svg.png"> <meta property="og:image:width" content="640"> <meta property="og:image:height" content="451"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Mutual information - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Mutual_information"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Mutual_information&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Mutual_information"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Mutual_information rootpage-Mutual_information skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" title="Main menu" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia"><span>Contents</span></a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events"><span>Current events</span></a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x"><span>Random article</span></a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works"><span>About Wikipedia</span></a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia"><span>Contact us</span></a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia"><span>Help</span></a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia"><span>Learn to edit</span></a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors"><span>Community portal</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia"><span>Upload file</span></a></li><li id="n-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages"><span>Special pages</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=en.wikipedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Mutual+information" title="You are encouraged to create an account and log in; however, it is not mandatory" class=""><span>Create account</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Mutual+information" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=en.wikipedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Mutual+information" title="You are encouraged to create an account and log in; however, it is not mandatory"><span class="vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd"></span> <span>Create account</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Mutual+information" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing"><span>learn more</span></a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y"><span>Contributions</span></a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n"><span>Talk</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-Definition" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Definition"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Definition</span> </div> </a> <button aria-controls="toc-Definition-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Definition subsection</span> </button> <ul id="toc-Definition-sublist" class="vector-toc-list"> <li id="toc-In_terms_of_PMFs_for_discrete_distributions" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#In_terms_of_PMFs_for_discrete_distributions"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.1</span> <span>In terms of PMFs for discrete distributions</span> </div> </a> <ul id="toc-In_terms_of_PMFs_for_discrete_distributions-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-In_terms_of_PDFs_for_continuous_distributions" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#In_terms_of_PDFs_for_continuous_distributions"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.2</span> <span>In terms of PDFs for continuous distributions</span> </div> </a> <ul id="toc-In_terms_of_PDFs_for_continuous_distributions-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Motivation" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Motivation"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Motivation</span> </div> </a> <ul id="toc-Motivation-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Properties" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Properties"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Properties</span> </div> </a> <button aria-controls="toc-Properties-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Properties subsection</span> </button> <ul id="toc-Properties-sublist" class="vector-toc-list"> <li id="toc-Nonnegativity" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Nonnegativity"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Nonnegativity</span> </div> </a> <ul id="toc-Nonnegativity-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Symmetry" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Symmetry"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.2</span> <span>Symmetry</span> </div> </a> <ul id="toc-Symmetry-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Supermodularity_under_independence" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Supermodularity_under_independence"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.3</span> <span>Supermodularity under independence</span> </div> </a> <ul id="toc-Supermodularity_under_independence-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Relation_to_conditional_and_joint_entropy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Relation_to_conditional_and_joint_entropy"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.4</span> <span>Relation to conditional and joint entropy</span> </div> </a> <ul id="toc-Relation_to_conditional_and_joint_entropy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Relation_to_Kullback–Leibler_divergence" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Relation_to_Kullback–Leibler_divergence"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.5</span> <span>Relation to Kullback–Leibler divergence</span> </div> </a> <ul id="toc-Relation_to_Kullback–Leibler_divergence-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Bayesian_estimation_of_mutual_information" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Bayesian_estimation_of_mutual_information"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.6</span> <span>Bayesian estimation of mutual information</span> </div> </a> <ul id="toc-Bayesian_estimation_of_mutual_information-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Independence_assumptions" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Independence_assumptions"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.7</span> <span>Independence assumptions</span> </div> </a> <ul id="toc-Independence_assumptions-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Variations" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Variations"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Variations</span> </div> </a> <button aria-controls="toc-Variations-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Variations subsection</span> </button> <ul id="toc-Variations-sublist" class="vector-toc-list"> <li id="toc-Metric" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Metric"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.1</span> <span>Metric</span> </div> </a> <ul id="toc-Metric-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Conditional_mutual_information" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Conditional_mutual_information"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.2</span> <span>Conditional mutual information</span> </div> </a> <ul id="toc-Conditional_mutual_information-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Interaction_information" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Interaction_information"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.3</span> <span>Interaction information</span> </div> </a> <ul id="toc-Interaction_information-sublist" class="vector-toc-list"> <li id="toc-Multivariate_statistical_independence" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Multivariate_statistical_independence"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.3.1</span> <span>Multivariate statistical independence</span> </div> </a> <ul id="toc-Multivariate_statistical_independence-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Applications" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Applications"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.3.2</span> <span>Applications</span> </div> </a> <ul id="toc-Applications-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Directed_information" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Directed_information"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4</span> <span>Directed information</span> </div> </a> <ul id="toc-Directed_information-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Normalized_variants" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Normalized_variants"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.5</span> <span>Normalized variants</span> </div> </a> <ul id="toc-Normalized_variants-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Weighted_variants" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Weighted_variants"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.6</span> <span>Weighted variants</span> </div> </a> <ul id="toc-Weighted_variants-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Adjusted_mutual_information" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Adjusted_mutual_information"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.7</span> <span>Adjusted mutual information</span> </div> </a> <ul id="toc-Adjusted_mutual_information-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Absolute_mutual_information" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Absolute_mutual_information"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.8</span> <span>Absolute mutual information</span> </div> </a> <ul id="toc-Absolute_mutual_information-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Linear_correlation" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Linear_correlation"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.9</span> <span>Linear correlation</span> </div> </a> <ul id="toc-Linear_correlation-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-For_discrete_data" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#For_discrete_data"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.10</span> <span>For discrete data</span> </div> </a> <ul id="toc-For_discrete_data-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Applications_2" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Applications_2"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Applications</span> </div> </a> <ul id="toc-Applications_2-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>See also</span> </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Notes" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Notes"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>Notes</span> </div> </a> <ul id="toc-Notes-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" title="Table of Contents" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Mutual information</span></h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 20 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-20" aria-hidden="true" ><span class="vector-icon mw-ui-icon-language-progressive mw-ui-icon-wikimedia-language-progressive"></span> <span class="vector-dropdown-label-text">20 languages</span> </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D9%85%D8%B9%D9%84%D9%88%D9%85%D8%A7%D8%AA_%D9%85%D8%AA%D8%A8%D8%A7%D8%AF%D9%84%D8%A9" title="معلومات متبادلة – Arabic" lang="ar" hreflang="ar" data-title="معلومات متبادلة" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target"><span>العربية</span></a></li><li class="interlanguage-link interwiki-bar mw-list-item"><a href="https://bar.wikipedia.org/wiki/Transinformation" title="Transinformation – Bavarian" lang="bar" hreflang="bar" data-title="Transinformation" data-language-autonym="Boarisch" data-language-local-name="Bavarian" class="interlanguage-link-target"><span>Boarisch</span></a></li><li class="interlanguage-link interwiki-cs mw-list-item"><a href="https://cs.wikipedia.org/wiki/Vz%C3%A1jemn%C3%A1_informace" title="Vzájemná informace – Czech" lang="cs" hreflang="cs" data-title="Vzájemná informace" data-language-autonym="Čeština" data-language-local-name="Czech" class="interlanguage-link-target"><span>Čeština</span></a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Transinformation" title="Transinformation – German" lang="de" hreflang="de" data-title="Transinformation" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target"><span>Deutsch</span></a></li><li class="interlanguage-link interwiki-el mw-list-item"><a href="https://el.wikipedia.org/wiki/%CE%91%CE%BC%CE%BF%CE%B9%CE%B2%CE%B1%CE%AF%CE%B1_%CF%80%CE%BB%CE%B7%CF%81%CE%BF%CF%86%CE%BF%CF%81%CE%AF%CE%B1" title="Αμοιβαία πληροφορία – Greek" lang="el" hreflang="el" data-title="Αμοιβαία πληροφορία" data-language-autonym="Ελληνικά" data-language-local-name="Greek" class="interlanguage-link-target"><span>Ελληνικά</span></a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Informaci%C3%B3n_mutua" title="Información mutua – Spanish" lang="es" hreflang="es" data-title="Información mutua" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target"><span>Español</span></a></li><li class="interlanguage-link interwiki-eu mw-list-item"><a href="https://eu.wikipedia.org/wiki/Elkarrekiko_informazio" title="Elkarrekiko informazio – Basque" lang="eu" hreflang="eu" data-title="Elkarrekiko informazio" data-language-autonym="Euskara" data-language-local-name="Basque" class="interlanguage-link-target"><span>Euskara</span></a></li><li class="interlanguage-link interwiki-fa mw-list-item"><a href="https://fa.wikipedia.org/wiki/%D8%A7%D8%B7%D9%84%D8%A7%D8%B9%D8%A7%D8%AA_%D9%85%D8%AA%D9%82%D8%A7%D8%A8%D9%84" title="اطلاعات متقابل – Persian" lang="fa" hreflang="fa" data-title="اطلاعات متقابل" data-language-autonym="فارسی" data-language-local-name="Persian" class="interlanguage-link-target"><span>فارسی</span></a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/Information_mutuelle" title="Information mutuelle – French" lang="fr" hreflang="fr" data-title="Information mutuelle" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target"><span>Français</span></a></li><li class="interlanguage-link interwiki-ko mw-list-item"><a href="https://ko.wikipedia.org/wiki/%EC%83%81%ED%98%B8%EC%A0%95%EB%B3%B4" title="상호정보 – Korean" lang="ko" hreflang="ko" data-title="상호정보" data-language-autonym="한국어" data-language-local-name="Korean" class="interlanguage-link-target"><span>한국어</span></a></li><li class="interlanguage-link interwiki-it mw-list-item"><a href="https://it.wikipedia.org/wiki/Informazione_mutua" title="Informazione mutua – Italian" lang="it" hreflang="it" data-title="Informazione mutua" data-language-autonym="Italiano" data-language-local-name="Italian" class="interlanguage-link-target"><span>Italiano</span></a></li><li class="interlanguage-link interwiki-he mw-list-item"><a href="https://he.wikipedia.org/wiki/%D7%90%D7%99%D7%A0%D7%A4%D7%95%D7%A8%D7%9E%D7%A6%D7%99%D7%94_%D7%94%D7%93%D7%93%D7%99%D7%AA" title="אינפורמציה הדדית – Hebrew" lang="he" hreflang="he" data-title="אינפורמציה הדדית" data-language-autonym="עברית" data-language-local-name="Hebrew" class="interlanguage-link-target"><span>עברית</span></a></li><li class="interlanguage-link interwiki-ja mw-list-item"><a href="https://ja.wikipedia.org/wiki/%E7%9B%B8%E4%BA%92%E6%83%85%E5%A0%B1%E9%87%8F" title="相互情報量 – Japanese" lang="ja" hreflang="ja" data-title="相互情報量" data-language-autonym="日本語" data-language-local-name="Japanese" class="interlanguage-link-target"><span>日本語</span></a></li><li class="interlanguage-link interwiki-pl mw-list-item"><a href="https://pl.wikipedia.org/wiki/Informacja_wzajemna" title="Informacja wzajemna – Polish" lang="pl" hreflang="pl" data-title="Informacja wzajemna" data-language-autonym="Polski" data-language-local-name="Polish" class="interlanguage-link-target"><span>Polski</span></a></li><li class="interlanguage-link interwiki-pt mw-list-item"><a href="https://pt.wikipedia.org/wiki/Informa%C3%A7%C3%A3o_m%C3%BAtua" title="Informação mútua – Portuguese" lang="pt" hreflang="pt" data-title="Informação mútua" data-language-autonym="Português" data-language-local-name="Portuguese" class="interlanguage-link-target"><span>Português</span></a></li><li class="interlanguage-link interwiki-ru mw-list-item"><a href="https://ru.wikipedia.org/wiki/%D0%92%D0%B7%D0%B0%D0%B8%D0%BC%D0%BD%D0%B0%D1%8F_%D0%B8%D0%BD%D1%84%D0%BE%D1%80%D0%BC%D0%B0%D1%86%D0%B8%D1%8F" title="Взаимная информация – Russian" lang="ru" hreflang="ru" data-title="Взаимная информация" data-language-autonym="Русский" data-language-local-name="Russian" class="interlanguage-link-target"><span>Русский</span></a></li><li class="interlanguage-link interwiki-simple mw-list-item"><a href="https://simple.wikipedia.org/wiki/Mutual_information" title="Mutual information – Simple English" lang="en-simple" hreflang="en-simple" data-title="Mutual information" data-language-autonym="Simple English" data-language-local-name="Simple English" class="interlanguage-link-target"><span>Simple English</span></a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%92%D0%B7%D0%B0%D1%94%D0%BC%D0%BD%D0%B0_%D1%96%D0%BD%D1%84%D0%BE%D1%80%D0%BC%D0%B0%D1%86%D1%96%D1%8F" title="Взаємна інформація – Ukrainian" lang="uk" hreflang="uk" data-title="Взаємна інформація" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target"><span>Українська</span></a></li><li class="interlanguage-link interwiki-zh-yue mw-list-item"><a href="https://zh-yue.wikipedia.org/wiki/%E7%9B%B8%E4%BA%92%E8%B3%87%E8%A8%8A" title="相互資訊 – Cantonese" lang="yue" hreflang="yue" data-title="相互資訊" data-language-autonym="粵語" data-language-local-name="Cantonese" class="interlanguage-link-target"><span>粵語</span></a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E4%BA%92%E4%BF%A1%E6%81%AF" title="互信息 – Chinese" lang="zh" hreflang="zh" data-title="互信息" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target"><span>中文</span></a></li> </ul> <div class="after-portlet after-portlet-lang"><span class="wb-langlinks-edit wb-langlinks-link"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q252973#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></span></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Mutual_information" title="View the content page [c]" accesskey="c"><span>Article</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Mutual_information" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t"><span>Talk</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Mutual_information"><span>Read</span></a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Mutual_information&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Mutual_information&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Mutual_information"><span>Read</span></a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Mutual_information&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Mutual_information&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Mutual_information" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Mutual_information" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-upload" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u"><span>Upload file</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Mutual_information&oldid=1270339045" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Mutual_information&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Mutual_information&id=1270339045&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FMutual_information"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FMutual_information"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Mutual_information&action=show-download-screen" title="Download this page as a PDF file"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Mutual_information&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q252973" title="Structured data on this page hosted by Wikidata [g]" accesskey="g"><span>Wikidata item</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Measure of dependence between two variables</div> <p class="mw-empty-elt"> </p> <style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1246091330">.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}</style><table class="sidebar nomobile nowraplinks hlist"><tbody><tr><th class="sidebar-title"><a href="/wiki/Information_theory" title="Information theory">Information theory</a></th></tr><tr><td class="sidebar-image"><span typeof="mw:File"><a href="/wiki/File:Binaryerasurechannel.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/2/23/Binaryerasurechannel.png/100px-Binaryerasurechannel.png" decoding="async" width="100" height="92" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/2/23/Binaryerasurechannel.png/150px-Binaryerasurechannel.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/2/23/Binaryerasurechannel.png/200px-Binaryerasurechannel.png 2x" data-file-width="666" data-file-height="610" /></a></span></td></tr><tr><td class="sidebar-content"> <ul><li><a href="/wiki/Entropy_(information_theory)" title="Entropy (information theory)">Entropy</a></li> <li><a href="/wiki/Differential_entropy" title="Differential entropy">Differential entropy</a></li> <li><a href="/wiki/Conditional_entropy" title="Conditional entropy">Conditional entropy</a></li> <li><a href="/wiki/Joint_entropy" title="Joint entropy">Joint entropy</a></li> <li><a class="mw-selflink selflink">Mutual information</a></li> <li><a href="/wiki/Directed_information" title="Directed information">Directed information</a></li> <li><a href="/wiki/Conditional_mutual_information" title="Conditional mutual information">Conditional mutual information</a></li> <li><a href="/wiki/Relative_entropy" class="mw-redirect" title="Relative entropy">Relative entropy</a></li> <li><a href="/wiki/Entropy_rate" title="Entropy rate">Entropy rate</a></li> <li><a href="/wiki/Limiting_density_of_discrete_points" title="Limiting density of discrete points">Limiting density of discrete points</a></li></ul></td> </tr><tr><td class="sidebar-content"> <ul><li><a href="/wiki/Asymptotic_equipartition_property" title="Asymptotic equipartition property">Asymptotic equipartition property</a></li> <li><a href="/wiki/Rate%E2%80%93distortion_theory" title="Rate–distortion theory">Rate–distortion theory</a></li></ul></td> </tr><tr><td class="sidebar-content"> <ul><li><a href="/wiki/Shannon%27s_source_coding_theorem" title="Shannon's source coding theorem">Shannon's source coding theorem</a></li> <li><a href="/wiki/Channel_capacity" title="Channel capacity">Channel capacity</a></li> <li><a href="/wiki/Noisy-channel_coding_theorem" title="Noisy-channel coding theorem">Noisy-channel coding theorem</a></li> <li><a href="/wiki/Shannon%E2%80%93Hartley_theorem" title="Shannon–Hartley theorem">Shannon–Hartley theorem</a></li></ul></td> </tr><tr><td class="sidebar-navbar"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Information_theory" title="Template:Information theory"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Information_theory" title="Template talk:Information theory"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Information_theory" title="Special:EditPage/Template:Information theory"><abbr title="Edit this template">e</abbr></a></li></ul></div></td></tr></tbody></table> <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:Entropy-mutual-information-relative-entropy-relation-diagram.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Entropy-mutual-information-relative-entropy-relation-diagram.svg/256px-Entropy-mutual-information-relative-entropy-relation-diagram.svg.png" decoding="async" width="256" height="180" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Entropy-mutual-information-relative-entropy-relation-diagram.svg/384px-Entropy-mutual-information-relative-entropy-relation-diagram.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Entropy-mutual-information-relative-entropy-relation-diagram.svg/512px-Entropy-mutual-information-relative-entropy-relation-diagram.svg.png 2x" data-file-width="744" data-file-height="524" /></a><figcaption><a href="/wiki/Venn_diagram" title="Venn diagram">Venn diagram</a> showing additive and subtractive relationships of various information measures associated with correlated variables <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>.<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> The area contained by either circle is the <a href="/wiki/Joint_entropy" title="Joint entropy">joint entropy</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/870bbc4cf58de95ebd325676f495ffda21a21c97" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.34ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X,Y)}"></span>. The circle on the left (red and violet) is the <a href="/wiki/Entropy_(information_theory)" title="Entropy (information theory)">individual entropy</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b3655e2f8fe1012c2cc749241584fdcc00bfa318" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.532ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X)}"></span>, with the red being the <a href="/wiki/Conditional_entropy" title="Conditional entropy">conditional entropy</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X\mid Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X\mid Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6a376b5ceee27a1d0a6a631dc0543bcc22890783" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.243ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X\mid Y)}"></span>. The circle on the right (blue and violet) is <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b65e884746ac4cde16228d02b6bdea42b7dccbd6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.326ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y)}"></span>, with the blue being <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y\mid X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y\mid X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bb22a6e097091b8fa931d58e15eafa972b479cf1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.243ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y\mid X)}"></span>. The violet is the mutual information <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7771931da3da8df3d83d58ab8d42715ebbb3fa4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.436ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)}"></span>.</figcaption></figure> <p>In <a href="/wiki/Probability_theory" title="Probability theory">probability theory</a> and <a href="/wiki/Information_theory" title="Information theory">information theory</a>, the <b>mutual information</b> (<b>MI</b>) of two <a href="/wiki/Random_variable" title="Random variable">random variables</a> is a measure of the mutual <a href="/wiki/Statistical_dependence" class="mw-redirect" title="Statistical dependence">dependence</a> between the two variables. More specifically, it quantifies the "<a href="/wiki/Information_content" title="Information content">amount of information</a>" (in <a href="/wiki/Units_of_information" title="Units of information">units</a> such as <a href="/wiki/Shannon_(unit)" title="Shannon (unit)">shannons</a> (<a href="/wiki/Bit" title="Bit">bits</a>), <a href="/wiki/Nat_(unit)" title="Nat (unit)">nats</a> or <a href="/wiki/Hartley_(unit)" title="Hartley (unit)">hartleys</a>) obtained about one random variable by observing the other random variable. The concept of mutual information is intimately linked to that of <a href="/wiki/Entropy_(information_theory)" title="Entropy (information theory)">entropy</a> of a random variable, a fundamental notion in information theory that quantifies the expected "amount of information" held in a random variable. </p><p>Not limited to real-valued random variables and linear dependence like the <a href="/wiki/Pearson_correlation_coefficient" title="Pearson correlation coefficient">correlation coefficient</a>, MI is more general and determines how different the <a href="/wiki/Joint_distribution" class="mw-redirect" title="Joint distribution">joint distribution</a> of the pair <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41f29b9537685f499713112d6802e811cbf51bba" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.597ex; height:2.843ex;" alt="{\displaystyle (X,Y)}"></span> is from the product of the marginal distributions of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>. MI is the <a href="/wiki/Expected_value" title="Expected value">expected value</a> of the <a href="/wiki/Pointwise_mutual_information" title="Pointwise mutual information">pointwise mutual information</a> (PMI). </p><p>The quantity was defined and analyzed by <a href="/wiki/Claude_Shannon" title="Claude Shannon">Claude Shannon</a> in his landmark paper "<a href="/wiki/A_Mathematical_Theory_of_Communication" title="A Mathematical Theory of Communication">A Mathematical Theory of Communication</a>", although he did not call it "mutual information". This term was coined later by <a href="/wiki/Robert_Fano" title="Robert Fano">Robert Fano</a>.<sup id="cite_ref-2" class="reference"><a href="#cite_note-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> Mutual Information is also known as <a href="/wiki/Information_gain" class="mw-redirect" title="Information gain">information gain</a>. </p> <meta property="mw:PageProp/toc" /> <div class="mw-heading mw-heading2"><h2 id="Definition">Definition</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=1" title="Edit section: Definition"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Let <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41f29b9537685f499713112d6802e811cbf51bba" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.597ex; height:2.843ex;" alt="{\displaystyle (X,Y)}"></span> be a pair of <a href="/wiki/Random_variable" title="Random variable">random variables</a> with values over the space <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {X}}\times {\mathcal {Y}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>×</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\mathcal {X}}\times {\mathcal {Y}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/58e4752674707ea9f3ebd3c8e049f06828320405" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.375ex; height:2.509ex;" alt="{\displaystyle {\mathcal {X}}\times {\mathcal {Y}}}"></span>. If their joint distribution is <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{(X,Y)}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{(X,Y)}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c2869dd2b7407ec3d99e02c7fee23029dd5c01f8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:6.115ex; height:3.009ex;" alt="{\displaystyle P_{(X,Y)}}"></span> and the marginal distributions are <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{X}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{X}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8348dd8ce7e6f7f4778ee01fa5bdc7b828afd98c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.125ex; height:2.509ex;" alt="{\displaystyle P_{X}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e52a028cc165bfc94caedb9464a32c36bf49130a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.978ex; height:2.509ex;" alt="{\displaystyle P_{Y}}"></span>, the mutual information is defined as </p> <div class="equation-box" style="margin: 0 0 0 1.6em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <p><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X;Y)=D_{\mathrm {KL} }(P_{(X,Y)}\|P_{X}\otimes P_{Y})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">K</mi> <mi mathvariant="normal">L</mi> </mrow> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo>⊗</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X;Y)=D_{\mathrm {KL} }(P_{(X,Y)}\|P_{X}\otimes P_{Y})}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4b37af4015d70867b9c50b9cf82e0fc3598b070c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:33.36ex; height:3.176ex;" alt="{\displaystyle I(X;Y)=D_{\mathrm {KL} }(P_{(X,Y)}\|P_{X}\otimes P_{Y})}"></span> </p> </div> <p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D_{\mathrm {KL} }}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">K</mi> <mi mathvariant="normal">L</mi> </mrow> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D_{\mathrm {KL} }}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2ab26e8799fe9a2b74e4c49da80bb18eaff04da0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.462ex; height:2.509ex;" alt="{\displaystyle D_{\mathrm {KL} }}"></span> is the <a href="/wiki/Kullback%E2%80%93Leibler_divergence" title="Kullback–Leibler divergence">Kullback–Leibler divergence</a>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{X}\otimes P_{Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo>⊗</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{X}\otimes P_{Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/5a06f4a1fdb28d4a925f0d28c29866b51494b0a0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.943ex; height:2.509ex;" alt="{\displaystyle P_{X}\otimes P_{Y}}"></span> is the <a href="/wiki/Outer_product" title="Outer product">outer product</a> distribution which assigns probability <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{X}(x)\cdot P_{Y}(y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{X}(x)\cdot P_{Y}(y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/02d60fc35fb28fc248cba69ba04a462ade049781" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.886ex; height:2.843ex;" alt="{\displaystyle P_{X}(x)\cdot P_{Y}(y)}"></span> to each <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41cf50e4a314ca8e2c30964baa8d26e5be7a9386" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.328ex; height:2.843ex;" alt="{\displaystyle (x,y)}"></span>. </p><p>Notice, as per property of the <a href="/wiki/Kullback%E2%80%93Leibler_divergence" title="Kullback–Leibler divergence">Kullback–Leibler divergence</a>, that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/14b479006c523b8459baf3cf169e3f4a7bcd0771" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.768ex; height:2.843ex;" alt="{\displaystyle I(X;Y)}"></span> is equal to zero precisely when the joint distribution coincides with the product of the marginals, i.e. when <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> are independent (and hence observing <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> tells you nothing about <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span>). <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/14b479006c523b8459baf3cf169e3f4a7bcd0771" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.768ex; height:2.843ex;" alt="{\displaystyle I(X;Y)}"></span> is non-negative, it is a measure of the price for encoding <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41f29b9537685f499713112d6802e811cbf51bba" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.597ex; height:2.843ex;" alt="{\displaystyle (X,Y)}"></span> as a pair of independent random variables when in reality they are not. </p><p>If the <a href="/wiki/Natural_logarithm" title="Natural logarithm">natural logarithm</a> is used, the unit of mutual information is the <a href="/wiki/Nat_(unit)" title="Nat (unit)">nat</a>. If the <a href="/wiki/Logarithm" title="Logarithm">log base</a> 2 is used, the unit of mutual information is the <a href="/wiki/Shannon_(unit)" title="Shannon (unit)">shannon</a>, also known as the bit. If the <a href="/wiki/Logarithm" title="Logarithm">log base</a> 10 is used, the unit of mutual information is the <a href="/wiki/Hartley_(unit)" title="Hartley (unit)">hartley</a>, also known as the ban or the dit. </p> <div class="mw-heading mw-heading3"><h3 id="In_terms_of_PMFs_for_discrete_distributions">In terms of PMFs for discrete distributions</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=2" title="Edit section: In terms of PMFs for discrete distributions"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The mutual information of two jointly discrete random variables <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> is calculated as a double sum:<sup id="cite_ref-cover1991_3-0" class="reference"><a href="#cite_note-cover1991-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 20">: 20 </span></sup> </p> <div class="equation-box" style="margin: 0 0 0 1.6em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <style data-mw-deduplicate="TemplateStyles:r1266403038">.mw-parser-output table.numblk{border-collapse:collapse;border:none;margin-top:0;margin-right:0;margin-bottom:0}.mw-parser-output table.numblk>tbody>tr>td{vertical-align:middle;padding:0}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2){width:99%}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table{border-collapse:collapse;margin:0;border:none;width:100%}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table>tbody>tr:first-child>td:first-child,.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table>tbody>tr:first-child>td:last-child{padding:0 0.4ex}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table>tbody>tr:first-child>td:nth-child(2){width:100%;padding:0}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table>tbody>tr:last-child>td{padding:0}.mw-parser-output table.numblk>tbody>tr>td:last-child{font-weight:bold}.mw-parser-output table.numblk.numblk-raw-n>tbody>tr>td:last-child{font-weight:unset}.mw-parser-output table.numblk>tbody>tr>td:last-child::before{content:"("}.mw-parser-output table.numblk>tbody>tr>td:last-child::after{content:")"}.mw-parser-output table.numblk.numblk-raw-n>tbody>tr>td:last-child::before,.mw-parser-output table.numblk.numblk-raw-n>tbody>tr>td:last-child::after{content:none}.mw-parser-output table.numblk>tbody>tr>td{border:none}.mw-parser-output table.numblk.numblk-border>tbody>tr>td{border:thin solid}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table>tbody>tr:first-child>td{border:none}.mw-parser-output table.numblk.numblk-border>tbody>tr>td:nth-child(2)>table>tbody>tr:first-child>td{border:thin solid}.mw-parser-output table.numblk>tbody>tr>td:nth-child(2)>table>tbody>tr:last-child>td{border-left:none;border-right:none;border-bottom:none}.mw-parser-output table.numblk.numblk-border>tbody>tr>td:nth-child(2)>table>tbody>tr:last-child>td{border-left:thin solid;border-right:thin solid;border-bottom:thin solid}.mw-parser-output table.numblk:target{color:var(--color-base,#202122);background-color:#cfe8fd}@media screen{html.skin-theme-clientpref-night .mw-parser-output table.numblk:target{color:var(--color-base,#eaecf0);background-color:#301702}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output table.numblk:target{color:var(--color-base,#eaecf0);background-color:#301702}}</style><table role="presentation" class="numblk" style="margin-left: 0em;"><tbody><tr><td class="nowrap"><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{P_{(X,Y)}(x,y)\log \left({\frac {P_{(X,Y)}(x,y)}{P_{X}(x)\,P_{Y}(y)}}\right)},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>)</mo> </mrow> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{P_{(X,Y)}(x,y)\log \left({\frac {P_{(X,Y)}(x,y)}{P_{X}(x)\,P_{Y}(y)}}\right)},}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1030462a874c1160206cf9347302067e20dbfb9a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:50.587ex; height:7.676ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{P_{(X,Y)}(x,y)\log \left({\frac {P_{(X,Y)}(x,y)}{P_{X}(x)\,P_{Y}(y)}}\right)},}"></span> </td> <td></td> <td class="nowrap"><span id="math_Eq.1" class="reference nourlexpansion" style="font-weight:bold;">Eq.1</span></td></tr></tbody></table> </div> <p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{(X,Y)}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{(X,Y)}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c2869dd2b7407ec3d99e02c7fee23029dd5c01f8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:6.115ex; height:3.009ex;" alt="{\displaystyle P_{(X,Y)}}"></span> is the <a href="/wiki/Joint_distribution" class="mw-redirect" title="Joint distribution">joint probability <i>mass</i> function</a> of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{X}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{X}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8348dd8ce7e6f7f4778ee01fa5bdc7b828afd98c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.125ex; height:2.509ex;" alt="{\displaystyle P_{X}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e52a028cc165bfc94caedb9464a32c36bf49130a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.978ex; height:2.509ex;" alt="{\displaystyle P_{Y}}"></span> are the <a href="/wiki/Marginal_probability" class="mw-redirect" title="Marginal probability">marginal probability</a> mass functions of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> respectively. </p> <div class="mw-heading mw-heading3"><h3 id="In_terms_of_PDFs_for_continuous_distributions">In terms of PDFs for continuous distributions</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=3" title="Edit section: In terms of PDFs for continuous distributions"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>In the case of jointly continuous random variables, the double sum is replaced by a <a href="/wiki/Double_integral" class="mw-redirect" title="Double integral">double integral</a>:<sup id="cite_ref-cover1991_3-1" class="reference"><a href="#cite_note-cover1991-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 251">: 251 </span></sup> </p> <div class="equation-box" style="margin: 0 0 0 1.6em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1266403038"><table role="presentation" class="numblk" style="margin-left: 0em;"><tbody><tr><td class="nowrap"><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\int _{\mathcal {Y}}\int _{\mathcal {X}}{P_{(X,Y)}(x,y)\log {\left({\frac {P_{(X,Y)}(x,y)}{P_{X}(x)\,P_{Y}(y)}}\right)}}\;dx\,dy,}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </msub> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>(</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>)</mo> </mrow> </mrow> </mrow> <mspace width="thickmathspace" /> <mi>d</mi> <mi>x</mi> <mspace width="thinmathspace" /> <mi>d</mi> <mi>y</mi> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\int _{\mathcal {Y}}\int _{\mathcal {X}}{P_{(X,Y)}(x,y)\log {\left({\frac {P_{(X,Y)}(x,y)}{P_{X}(x)\,P_{Y}(y)}}\right)}}\;dx\,dy,}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3f2b5ef93bc8c87f2ca4fe15e7a914f9fd163976" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:55.755ex; height:7.509ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\int _{\mathcal {Y}}\int _{\mathcal {X}}{P_{(X,Y)}(x,y)\log {\left({\frac {P_{(X,Y)}(x,y)}{P_{X}(x)\,P_{Y}(y)}}\right)}}\;dx\,dy,}"></span> </td> <td></td> <td class="nowrap"><span id="math_Eq.2" class="reference nourlexpansion" style="font-weight:bold;">Eq.2</span></td></tr></tbody></table> </div> <p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{(X,Y)}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{(X,Y)}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c2869dd2b7407ec3d99e02c7fee23029dd5c01f8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:6.115ex; height:3.009ex;" alt="{\displaystyle P_{(X,Y)}}"></span> is now the joint probability <i>density</i> function of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{X}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{X}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8348dd8ce7e6f7f4778ee01fa5bdc7b828afd98c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.125ex; height:2.509ex;" alt="{\displaystyle P_{X}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e52a028cc165bfc94caedb9464a32c36bf49130a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.978ex; height:2.509ex;" alt="{\displaystyle P_{Y}}"></span> are the marginal probability density functions of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> respectively. </p> <div class="mw-heading mw-heading2"><h2 id="Motivation">Motivation</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=4" title="Edit section: Motivation"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Intuitively, mutual information measures the information that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> share: It measures how much knowing one of these variables reduces uncertainty about the other. For example, if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> are independent, then knowing <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> does not give any information about <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> and vice versa, so their mutual information is zero. At the other extreme, if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> is a deterministic function of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> is a deterministic function of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> then all information conveyed by <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> is shared with <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>: knowing <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> determines the value of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> and vice versa. As a result, the mutual information is the same as the uncertainty contained in <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> (or <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span>) alone, namely the <a href="/wiki/Information_entropy" class="mw-redirect" title="Information entropy">entropy</a> of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> (or <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span>). A very special case of this is when <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> are the same random variable. </p><p>Mutual information is a measure of the inherent dependence expressed in the <a href="/wiki/Joint_distribution" class="mw-redirect" title="Joint distribution">joint distribution</a> of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> relative to the marginal distribution of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> under the assumption of independence. Mutual information therefore measures dependence in the following sense: <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/049558e6650dbcedaeff54372222d72ece9d04dc" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.697ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)=0}"></span> <a href="/wiki/If_and_only_if" title="If and only if">if and only if</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> are independent random variables. This is easy to see in one direction: if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> are independent, then <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{(X,Y)}(x,y)=p_{X}(x)\cdot p_{Y}(y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{(X,Y)}(x,y)=p_{X}(x)\cdot p_{Y}(y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a416701e5e1aba73e47b2c9325b54a5221162a8f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; margin-left: -0.089ex; width:27.549ex; height:3.176ex;" alt="{\displaystyle p_{(X,Y)}(x,y)=p_{X}(x)\cdot p_{Y}(y)}"></span>, and therefore: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \log {\left({\frac {p_{(X,Y)}(x,y)}{p_{X}(x)\,p_{Y}(y)}}\right)}=\log 1=0.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>(</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>)</mo> </mrow> </mrow> <mo>=</mo> <mi>log</mi> <mo>⁡</mo> <mn>1</mn> <mo>=</mo> <mn>0.</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \log {\left({\frac {p_{(X,Y)}(x,y)}{p_{X}(x)\,p_{Y}(y)}}\right)}=\log 1=0.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9e6f5a4ad28d98e4f1905cbf51a9ef6a50334128" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:32.352ex; height:7.509ex;" alt="{\displaystyle \log {\left({\frac {p_{(X,Y)}(x,y)}{p_{X}(x)\,p_{Y}(y)}}\right)}=\log 1=0.}"></span></dd></dl> <p>Moreover, mutual information is nonnegative (i.e. <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)\geq 0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>≥</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)\geq 0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/379a16ab656c2790190a9987614fd43b824e051b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.697ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)\geq 0}"></span> see below) and <a href="/wiki/Symmetric_function" title="Symmetric function">symmetric</a> (i.e. <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\operatorname {I} (Y;X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\operatorname {I} (Y;X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fa96bc32dab6af6e773d378d5e9ca968cc94fdae" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:17.971ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\operatorname {I} (Y;X)}"></span> see below). </p> <div class="mw-heading mw-heading2"><h2 id="Properties">Properties</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=5" title="Edit section: Properties"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Nonnegativity">Nonnegativity</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=6" title="Edit section: Nonnegativity"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Using <a href="/wiki/Jensen%27s_inequality" title="Jensen's inequality">Jensen's inequality</a> on the definition of mutual information we can show that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7771931da3da8df3d83d58ab8d42715ebbb3fa4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.436ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)}"></span> is non-negative, i.e.<sup id="cite_ref-cover1991_3-2" class="reference"><a href="#cite_note-cover1991-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 28">: 28 </span></sup> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)\geq 0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>≥</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)\geq 0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/379a16ab656c2790190a9987614fd43b824e051b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.697ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)\geq 0}"></span></dd></dl> <div class="mw-heading mw-heading3"><h3 id="Symmetry">Symmetry</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=7" title="Edit section: Symmetry"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\operatorname {I} (Y;X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\operatorname {I} (Y;X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fa96bc32dab6af6e773d378d5e9ca968cc94fdae" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:17.971ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\operatorname {I} (Y;X)}"></span></dd></dl> <p>The proof is given considering the relationship with entropy, as shown below. </p> <div class="mw-heading mw-heading3"><h3 id="Supermodularity_under_independence">Supermodularity under independence</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=8" title="Edit section: Supermodularity under independence"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>If <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle C}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>C</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle C}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4fc55753007cd3c18576f7933f6f089196732029" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.766ex; height:2.176ex;" alt="{\displaystyle C}"></span> is independent of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (A,B)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>A</mi> <mo>,</mo> <mi>B</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (A,B)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ce67314185650d6f0deba39db7dcec9378f4d4d1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.35ex; height:2.843ex;" alt="{\displaystyle (A,B)}"></span>, then </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (Y;A,B,C)-\operatorname {I} (Y;A,B)\geq \operatorname {I} (Y;A,C)-\operatorname {I} (Y;A)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>A</mi> <mo>,</mo> <mi>B</mi> <mo>,</mo> <mi>C</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>A</mi> <mo>,</mo> <mi>B</mi> <mo stretchy="false">)</mo> <mo>≥</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>A</mi> <mo>,</mo> <mi>C</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>A</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (Y;A,B,C)-\operatorname {I} (Y;A,B)\geq \operatorname {I} (Y;A,C)-\operatorname {I} (Y;A)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7d7af8ef99eb7729a94c8848447d885dc5932c30" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:48.773ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (Y;A,B,C)-\operatorname {I} (Y;A,B)\geq \operatorname {I} (Y;A,C)-\operatorname {I} (Y;A)}"></span>.<sup id="cite_ref-4" class="reference"><a href="#cite_note-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup></dd></dl> <div class="mw-heading mw-heading3"><h3 id="Relation_to_conditional_and_joint_entropy">Relation to conditional and joint entropy</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=9" title="Edit section: Relation to conditional and joint entropy"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Mutual information can be equivalently expressed as: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&{}\equiv \mathrm {H} (X)-\mathrm {H} (X\mid Y)\\&{}\equiv \mathrm {H} (Y)-\mathrm {H} (Y\mid X)\\&{}\equiv \mathrm {H} (X)+\mathrm {H} (Y)-\mathrm {H} (X,Y)\\&{}\equiv \mathrm {H} (X,Y)-\mathrm {H} (X\mid Y)-\mathrm {H} (Y\mid X)\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>≡</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>≡</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>≡</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>≡</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&{}\equiv \mathrm {H} (X)-\mathrm {H} (X\mid Y)\\&{}\equiv \mathrm {H} (Y)-\mathrm {H} (Y\mid X)\\&{}\equiv \mathrm {H} (X)+\mathrm {H} (Y)-\mathrm {H} (X,Y)\\&{}\equiv \mathrm {H} (X,Y)-\mathrm {H} (X\mid Y)-\mathrm {H} (Y\mid X)\end{aligned}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/805ea31b89caeaf4bfdf747aabf4b26ac78e8df2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -5.671ex; width:43.792ex; height:12.509ex;" alt="{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&{}\equiv \mathrm {H} (X)-\mathrm {H} (X\mid Y)\\&{}\equiv \mathrm {H} (Y)-\mathrm {H} (Y\mid X)\\&{}\equiv \mathrm {H} (X)+\mathrm {H} (Y)-\mathrm {H} (X,Y)\\&{}\equiv \mathrm {H} (X,Y)-\mathrm {H} (X\mid Y)-\mathrm {H} (Y\mid X)\end{aligned}}}"></span></dd></dl> <p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b3655e2f8fe1012c2cc749241584fdcc00bfa318" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.532ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X)}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b65e884746ac4cde16228d02b6bdea42b7dccbd6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.326ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y)}"></span> are the marginal <a href="/wiki/Information_entropy" class="mw-redirect" title="Information entropy">entropies</a>, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X\mid Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X\mid Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6a376b5ceee27a1d0a6a631dc0543bcc22890783" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.243ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X\mid Y)}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y\mid X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y\mid X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bb22a6e097091b8fa931d58e15eafa972b479cf1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.243ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y\mid X)}"></span> are the <a href="/wiki/Conditional_entropy" title="Conditional entropy">conditional entropies</a>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/870bbc4cf58de95ebd325676f495ffda21a21c97" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.34ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X,Y)}"></span> is the <a href="/wiki/Joint_entropy" title="Joint entropy">joint entropy</a> of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>. </p><p>Notice the analogy to the union, difference, and intersection of two sets: in this respect, all the formulas given above are apparent from the Venn diagram reported at the beginning of the article. </p><p>In terms of a communication channel in which the output <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> is a noisy version of the input <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span>, these relations are summarised in the figure: </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Figchannel2017ab.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b5/Figchannel2017ab.svg/220px-Figchannel2017ab.svg.png" decoding="async" width="220" height="103" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b5/Figchannel2017ab.svg/330px-Figchannel2017ab.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b5/Figchannel2017ab.svg/440px-Figchannel2017ab.svg.png 2x" data-file-width="800" data-file-height="374" /></a><figcaption>The relationships between information theoretic quantities</figcaption></figure> <p>Because <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7771931da3da8df3d83d58ab8d42715ebbb3fa4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.436ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)}"></span> is non-negative, consequently, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (X)\geq \mathrm {H} (X\mid Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>≥</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (X)\geq \mathrm {H} (X\mid Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/688078e5214c531db18ebab50dc14b60c56ec693" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:17.874ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (X)\geq \mathrm {H} (X\mid Y)}"></span>. Here we give the detailed deduction of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\mathrm {H} (Y)-\mathrm {H} (Y\mid X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\mathrm {H} (Y)-\mathrm {H} (Y\mid X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/90e959fb1c4662f5af6bd8345aa366170a815b7b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:27.944ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\mathrm {H} (Y)-\mathrm {H} (Y\mid X)}"></span> for the case of jointly discrete random variables: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log {\frac {p_{(X,Y)}(x,y)}{p_{X}(x)p_{Y}(y)}}\\&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log {\frac {p_{(X,Y)}(x,y)}{p_{X}(x)}}-\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log p_{Y}(y)\\&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{X}(x)p_{Y\mid X=x}(y)\log p_{Y\mid X=x}(y)-\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log p_{Y}(y)\\&{}=\sum _{x\in {\mathcal {X}}}p_{X}(x)\left(\sum _{y\in {\mathcal {Y}}}p_{Y\mid X=x}(y)\log p_{Y\mid X=x}(y)\right)-\sum _{y\in {\mathcal {Y}}}\left(\sum _{x\in {\mathcal {X}}}p_{(X,Y)}(x,y)\right)\log p_{Y}(y)\\&{}=-\sum _{x\in {\mathcal {X}}}p_{X}(x)\mathrm {H} (Y\mid X=x)-\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\log p_{Y}(y)\\&{}=-\mathrm {H} (Y\mid X)+\mathrm {H} (Y)\\&{}=\mathrm {H} (Y)-\mathrm {H} (Y\mid X).\\\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>,</mo> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>,</mo> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>−</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>,</mo> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>,</mo> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo>=</mo> <mi>x</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo>=</mo> <mi>x</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> <mo>,</mo> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mrow> <mo>(</mo> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo>=</mo> <mi>x</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo>=</mo> <mi>x</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mo>)</mo> </mrow> <mo>−</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <mrow> <mo>(</mo> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mo>)</mo> </mrow> <mi>log</mi> <mo>⁡</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <mo>−</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo>=</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>−</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>.</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log {\frac {p_{(X,Y)}(x,y)}{p_{X}(x)p_{Y}(y)}}\\&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log {\frac {p_{(X,Y)}(x,y)}{p_{X}(x)}}-\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log p_{Y}(y)\\&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{X}(x)p_{Y\mid X=x}(y)\log p_{Y\mid X=x}(y)-\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log p_{Y}(y)\\&{}=\sum _{x\in {\mathcal {X}}}p_{X}(x)\left(\sum _{y\in {\mathcal {Y}}}p_{Y\mid X=x}(y)\log p_{Y\mid X=x}(y)\right)-\sum _{y\in {\mathcal {Y}}}\left(\sum _{x\in {\mathcal {X}}}p_{(X,Y)}(x,y)\right)\log p_{Y}(y)\\&{}=-\sum _{x\in {\mathcal {X}}}p_{X}(x)\mathrm {H} (Y\mid X=x)-\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\log p_{Y}(y)\\&{}=-\mathrm {H} (Y\mid X)+\mathrm {H} (Y)\\&{}=\mathrm {H} (Y)-\mathrm {H} (Y\mid X).\\\end{aligned}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9c1877f635a7f1d6ba6a6cc0e65a9b7b1fec7b43" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -20.005ex; width:86.427ex; height:41.176ex;" alt="{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log {\frac {p_{(X,Y)}(x,y)}{p_{X}(x)p_{Y}(y)}}\\&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log {\frac {p_{(X,Y)}(x,y)}{p_{X}(x)}}-\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log p_{Y}(y)\\&{}=\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{X}(x)p_{Y\mid X=x}(y)\log p_{Y\mid X=x}(y)-\sum _{x\in {\mathcal {X}},y\in {\mathcal {Y}}}p_{(X,Y)}(x,y)\log p_{Y}(y)\\&{}=\sum _{x\in {\mathcal {X}}}p_{X}(x)\left(\sum _{y\in {\mathcal {Y}}}p_{Y\mid X=x}(y)\log p_{Y\mid X=x}(y)\right)-\sum _{y\in {\mathcal {Y}}}\left(\sum _{x\in {\mathcal {X}}}p_{(X,Y)}(x,y)\right)\log p_{Y}(y)\\&{}=-\sum _{x\in {\mathcal {X}}}p_{X}(x)\mathrm {H} (Y\mid X=x)-\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\log p_{Y}(y)\\&{}=-\mathrm {H} (Y\mid X)+\mathrm {H} (Y)\\&{}=\mathrm {H} (Y)-\mathrm {H} (Y\mid X).\\\end{aligned}}}"></span></dd></dl> <p>The proofs of the other identities above are similar. The proof of the general case (not just discrete) is similar, with integrals replacing sums. </p><p>Intuitively, if entropy <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b65e884746ac4cde16228d02b6bdea42b7dccbd6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.326ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y)}"></span> is regarded as a measure of uncertainty about a random variable, then <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y\mid X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y\mid X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bb22a6e097091b8fa931d58e15eafa972b479cf1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.243ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y\mid X)}"></span> is a measure of what <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> does <i>not</i> say about <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>. This is "the amount of uncertainty remaining about <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> after <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> is known", and thus the right side of the second of these equalities can be read as "the amount of uncertainty in <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>, minus the amount of uncertainty in <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> which remains after <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> is known", which is equivalent to "the amount of uncertainty in <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> which is removed by knowing <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span>". This corroborates the intuitive meaning of mutual information as the amount of information (that is, reduction in uncertainty) that knowing either variable provides about the other. </p><p>Note that in the discrete case <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y\mid Y)=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y\mid Y)=0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/181807446d4a05f6c27cdfbd4199f79147b1f5c8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.297ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y\mid Y)=0}"></span> and therefore <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {H} (Y)=\operatorname {I} (Y;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \mathrm {H} (Y)=\operatorname {I} (Y;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a45237d500199aa766c6a1ce2d7fe05366d2e96b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.654ex; height:2.843ex;" alt="{\displaystyle \mathrm {H} (Y)=\operatorname {I} (Y;Y)}"></span>. Thus <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (Y;Y)\geq \operatorname {I} (X;Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>≥</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (Y;Y)\geq \operatorname {I} (X;Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7b02e824143f3f582477bf3e94664f7d15243f1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:17.764ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (Y;Y)\geq \operatorname {I} (X;Y)}"></span>, and one can formulate the basic principle that a variable contains at least as much information about itself as any other variable can provide. </p> <div class="mw-heading mw-heading3"><h3 id="Relation_to_Kullback–Leibler_divergence"><span id="Relation_to_Kullback.E2.80.93Leibler_divergence"></span>Relation to Kullback–Leibler divergence</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=10" title="Edit section: Relation to Kullback–Leibler divergence"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>For jointly discrete or jointly continuous pairs <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41f29b9537685f499713112d6802e811cbf51bba" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.597ex; height:2.843ex;" alt="{\displaystyle (X,Y)}"></span>, mutual information is the <a href="/wiki/Kullback%E2%80%93Leibler_divergence" title="Kullback–Leibler divergence">Kullback–Leibler divergence</a> from the product of the <a href="/wiki/Marginal_distribution" title="Marginal distribution">marginal distributions</a>, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{X}\cdot p_{Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo>⋅</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{X}\cdot p_{Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c13ea37d9ea3db3c33471ad43b141fa4aeb3f2f8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:7.226ex; height:2.009ex;" alt="{\displaystyle p_{X}\cdot p_{Y}}"></span>, of the <a href="/wiki/Joint_distribution" class="mw-redirect" title="Joint distribution">joint distribution</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{(X,Y)}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{(X,Y)}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ea5ff436cc0c4f7128e6cef39ff2278f3029afe4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; margin-left: -0.089ex; width:5.882ex; height:2.509ex;" alt="{\displaystyle p_{(X,Y)}}"></span>, that is, </p> <div class="equation-box" style="margin: 0 0 0 1.6em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <p><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=D_{\text{KL}}\left(p_{(X,Y)}\parallel p_{X}p_{Y}\right)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>KL</mtext> </mrow> </msub> <mrow> <mo>(</mo> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo>∥</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> </mrow> <mo>)</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=D_{\text{KL}}\left(p_{(X,Y)}\parallel p_{X}p_{Y}\right)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c05555070284cd34756aa221f5efc544f06f56de" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:31.216ex; height:3.343ex;" alt="{\displaystyle \operatorname {I} (X;Y)=D_{\text{KL}}\left(p_{(X,Y)}\parallel p_{X}p_{Y}\right)}"></span> </p> </div> <p>Furthermore, let <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{(X,Y)}(x,y)=p_{X\mid Y=y}(x)*p_{Y}(y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo>=</mo> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>∗</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{(X,Y)}(x,y)=p_{X\mid Y=y}(x)*p_{Y}(y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/517e8dafc11dc189cd549c793c56dfbaac7105d0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; margin-left: -0.089ex; width:31.871ex; height:3.176ex;" alt="{\displaystyle p_{(X,Y)}(x,y)=p_{X\mid Y=y}(x)*p_{Y}(y)}"></span> be the conditional mass or density function. Then, we have the identity </p> <div class="equation-box" style="margin: 0 0 0 1.6em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <p><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\mathbb {E} _{Y}\left[D_{\text{KL}}\!\left(p_{X\mid Y}\parallel p_{X}\right)\right]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mrow> <mo>[</mo> <mrow> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>KL</mtext> </mrow> </msub> <mspace width="negativethinmathspace" /> <mrow> <mo>(</mo> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> </mrow> </msub> <mo>∥</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mrow> <mo>)</mo> </mrow> </mrow> <mo>]</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\mathbb {E} _{Y}\left[D_{\text{KL}}\!\left(p_{X\mid Y}\parallel p_{X}\right)\right]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/28338bdc75a2c5652cbb6a84e024d2b5ad74610c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:32.257ex; height:3.343ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\mathbb {E} _{Y}\left[D_{\text{KL}}\!\left(p_{X\mid Y}\parallel p_{X}\right)\right]}"></span> </p> </div> <p>The proof for jointly discrete random variables is as follows: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p_{(X,Y)}(x,y)\log \left({\frac {p_{(X,Y)}(x,y)}{p_{X}(x)\,p_{Y}(y)}}\right)}\\&=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}p_{X\mid Y=y}(x)p_{Y}(y)\log {\frac {p_{X\mid Y=y}(x)p_{Y}(y)}{p_{X}(x)p_{Y}(y)}}\\&=\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\sum _{x\in {\mathcal {X}}}p_{X\mid Y=y}(x)\log {\frac {p_{X\mid Y=y}(x)}{p_{X}(x)}}\\&=\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\;D_{\text{KL}}\!\left(p_{X\mid Y=y}\parallel p_{X}\right)\\&=\mathbb {E} _{Y}\left[D_{\text{KL}}\!\left(p_{X\mid Y}\parallel p_{X}\right)\right].\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>)</mo> </mrow> </mrow> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo>=</mo> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo>=</mo> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo>=</mo> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo>=</mo> <mi>y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mspace width="thickmathspace" /> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>KL</mtext> </mrow> </msub> <mspace width="negativethinmathspace" /> <mrow> <mo>(</mo> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo>=</mo> <mi>y</mi> </mrow> </msub> <mo>∥</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mrow> <mo>)</mo> </mrow> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mrow> <mo>[</mo> <mrow> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>KL</mtext> </mrow> </msub> <mspace width="negativethinmathspace" /> <mrow> <mo>(</mo> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> </mrow> </msub> <mo>∥</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mrow> <mo>)</mo> </mrow> </mrow> <mo>]</mo> </mrow> <mo>.</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p_{(X,Y)}(x,y)\log \left({\frac {p_{(X,Y)}(x,y)}{p_{X}(x)\,p_{Y}(y)}}\right)}\\&=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}p_{X\mid Y=y}(x)p_{Y}(y)\log {\frac {p_{X\mid Y=y}(x)p_{Y}(y)}{p_{X}(x)p_{Y}(y)}}\\&=\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\sum _{x\in {\mathcal {X}}}p_{X\mid Y=y}(x)\log {\frac {p_{X\mid Y=y}(x)}{p_{X}(x)}}\\&=\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\;D_{\text{KL}}\!\left(p_{X\mid Y=y}\parallel p_{X}\right)\\&=\mathbb {E} _{Y}\left[D_{\text{KL}}\!\left(p_{X\mid Y}\parallel p_{X}\right)\right].\end{aligned}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b644a60366facfab2ab84ca89da4068cc670ffea" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -15.671ex; width:54.096ex; height:32.509ex;" alt="{\displaystyle {\begin{aligned}\operatorname {I} (X;Y)&=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p_{(X,Y)}(x,y)\log \left({\frac {p_{(X,Y)}(x,y)}{p_{X}(x)\,p_{Y}(y)}}\right)}\\&=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}p_{X\mid Y=y}(x)p_{Y}(y)\log {\frac {p_{X\mid Y=y}(x)p_{Y}(y)}{p_{X}(x)p_{Y}(y)}}\\&=\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\sum _{x\in {\mathcal {X}}}p_{X\mid Y=y}(x)\log {\frac {p_{X\mid Y=y}(x)}{p_{X}(x)}}\\&=\sum _{y\in {\mathcal {Y}}}p_{Y}(y)\;D_{\text{KL}}\!\left(p_{X\mid Y=y}\parallel p_{X}\right)\\&=\mathbb {E} _{Y}\left[D_{\text{KL}}\!\left(p_{X\mid Y}\parallel p_{X}\right)\right].\end{aligned}}}"></span></dd></dl> <p>Similarly this identity can be established for jointly continuous random variables. </p><p>Note that here the Kullback–Leibler divergence involves integration over the values of the random variable <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> only, and the expression <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D_{\text{KL}}(p_{X\mid Y}\parallel p_{X})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>KL</mtext> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> </mrow> </msub> <mo>∥</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D_{\text{KL}}(p_{X\mid Y}\parallel p_{X})}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2b3d00c500923d35e585fc5371f84d76a1cda84c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:16.039ex; height:3.176ex;" alt="{\displaystyle D_{\text{KL}}(p_{X\mid Y}\parallel p_{X})}"></span> still denotes a random variable because <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> is random. Thus mutual information can also be understood as the <a href="/wiki/Expected_value" title="Expected value">expectation</a> of the Kullback–Leibler divergence of the <a href="/wiki/Univariate_distribution" title="Univariate distribution">univariate distribution</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{X}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{X}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4bc900e2770ed796e420eb5aa0852193a1919ae0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:2.891ex; height:2.009ex;" alt="{\displaystyle p_{X}}"></span> of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> from the <a href="/wiki/Conditional_distribution" class="mw-redirect" title="Conditional distribution">conditional distribution</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{X\mid Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{X\mid Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b479b78682756c24e3cf55c710896abc154d6c33" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; margin-left: -0.089ex; width:4.602ex; height:2.509ex;" alt="{\displaystyle p_{X\mid Y}}"></span> of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> given <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>: the more different the distributions <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{X\mid Y}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{X\mid Y}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b479b78682756c24e3cf55c710896abc154d6c33" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; margin-left: -0.089ex; width:4.602ex; height:2.509ex;" alt="{\displaystyle p_{X\mid Y}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{X}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{X}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4bc900e2770ed796e420eb5aa0852193a1919ae0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:2.891ex; height:2.009ex;" alt="{\displaystyle p_{X}}"></span> are on average, the greater the <a href="/wiki/Kullback%E2%80%93Leibler_divergence" title="Kullback–Leibler divergence">information gain</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Bayesian_estimation_of_mutual_information">Bayesian estimation of mutual information</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=11" title="Edit section: Bayesian estimation of mutual information"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>If samples from a joint distribution are available, a Bayesian approach can be used to estimate the mutual information of that distribution. The first work to do this, which also showed how to do Bayesian estimation of many other information-theoretic properties besides mutual information, was.<sup id="cite_ref-5" class="reference"><a href="#cite_note-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> Subsequent researchers have rederived <sup id="cite_ref-6" class="reference"><a href="#cite_note-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> and extended <sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup> this analysis. See <sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> for a recent paper based on a prior specifically tailored to estimation of mutual information per se. Besides, recently an estimation method accounting for continuous and multivariate outputs, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>, was proposed in .<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Independence_assumptions">Independence assumptions</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=12" title="Edit section: Independence assumptions"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The Kullback-Leibler divergence formulation of the mutual information is predicated on that one is interested in comparing <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089e91a1824e14cebc8e8d04dc652c61b3008e0a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:6.587ex; height:2.843ex;" alt="{\displaystyle p(x,y)}"></span> to the fully factorized <a href="/wiki/Outer_product" title="Outer product">outer product</a> <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x)\cdot p(y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mo>⋅</mo> <mi>p</mi> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x)\cdot p(y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d030dd64a08b2e264c564cb92c923e7e83bace95" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:10.211ex; height:2.843ex;" alt="{\displaystyle p(x)\cdot p(y)}"></span>. In many problems, such as <a href="/wiki/Non-negative_matrix_factorization" title="Non-negative matrix factorization">non-negative matrix factorization</a>, one is interested in less extreme factorizations; specifically, one wishes to compare <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089e91a1824e14cebc8e8d04dc652c61b3008e0a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:6.587ex; height:2.843ex;" alt="{\displaystyle p(x,y)}"></span> to a low-rank matrix approximation in some unknown variable <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/88b1e0c8e1be5ebe69d18a8010676fa42d7961e6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.664ex; height:1.676ex;" alt="{\displaystyle w}"></span>; that is, to what degree one might have </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)\approx \sum _{w}p^{\prime }(x,w)p^{\prime \prime }(w,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mo>≈</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>w</mi> </mrow> </munder> <msup> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-variant" mathvariant="normal">′</mi> </mrow> </msup> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>w</mi> <mo stretchy="false">)</mo> <msup> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-variant" mathvariant="normal">′</mi> <mi class="MJX-variant" mathvariant="normal">′</mi> </mrow> </msup> <mo stretchy="false">(</mo> <mi>w</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)\approx \sum _{w}p^{\prime }(x,w)p^{\prime \prime }(w,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/88336f45eef1b43afc1c47d083ef0a561c4fa134" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; margin-left: -0.089ex; width:29.088ex; height:5.509ex;" alt="{\displaystyle p(x,y)\approx \sum _{w}p^{\prime }(x,w)p^{\prime \prime }(w,y)}"></span></dd></dl> <p>Alternately, one might be interested in knowing how much more information <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089e91a1824e14cebc8e8d04dc652c61b3008e0a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:6.587ex; height:2.843ex;" alt="{\displaystyle p(x,y)}"></span> carries over its factorization. In such a case, the excess information that the full distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089e91a1824e14cebc8e8d04dc652c61b3008e0a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:6.587ex; height:2.843ex;" alt="{\displaystyle p(x,y)}"></span> carries over the matrix factorization is given by the Kullback-Leibler divergence </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} _{LRMA}=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p(x,y)\log {\left({\frac {p(x,y)}{\sum _{w}p^{\prime }(x,w)p^{\prime \prime }(w,y)}}\right)}},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi mathvariant="normal">I</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>L</mi> <mi>R</mi> <mi>M</mi> <mi>A</mi> </mrow> </msub> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>(</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>w</mi> </mrow> </munder> <msup> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-variant" mathvariant="normal">′</mi> </mrow> </msup> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>w</mi> <mo stretchy="false">)</mo> <msup> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-variant" mathvariant="normal">′</mi> <mi class="MJX-variant" mathvariant="normal">′</mi> </mrow> </msup> <mo stretchy="false">(</mo> <mi>w</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>)</mo> </mrow> </mrow> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} _{LRMA}=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p(x,y)\log {\left({\frac {p(x,y)}{\sum _{w}p^{\prime }(x,w)p^{\prime \prime }(w,y)}}\right)}},}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/86039b6a7deeea22a34e8313fc4e4ff62b6b5fa8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:52.047ex; height:7.176ex;" alt="{\displaystyle \operatorname {I} _{LRMA}=\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p(x,y)\log {\left({\frac {p(x,y)}{\sum _{w}p^{\prime }(x,w)p^{\prime \prime }(w,y)}}\right)}},}"></span></dd></dl> <p>The conventional definition of the mutual information is recovered in the extreme case that the process <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>W</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle W}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/54a9c4c547f4d6111f81946cad242b18298d70b7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.176ex;" alt="{\displaystyle W}"></span> has only one value for <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/88b1e0c8e1be5ebe69d18a8010676fa42d7961e6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.664ex; height:1.676ex;" alt="{\displaystyle w}"></span>. </p> <div class="mw-heading mw-heading2"><h2 id="Variations">Variations</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=13" title="Edit section: Variations"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Several variations on mutual information have been proposed to suit various needs. Among these are normalized variants and generalizations to more than two variables. </p> <div class="mw-heading mw-heading3"><h3 id="Metric">Metric</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=14" title="Edit section: Metric"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Many applications require a <a href="/wiki/Metric_(mathematics)" class="mw-redirect" title="Metric (mathematics)">metric</a>, that is, a distance measure between pairs of points. The quantity </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}d(X,Y)&=\mathrm {H} (X,Y)-\operatorname {I} (X;Y)\\&=\mathrm {H} (X)+\mathrm {H} (Y)-2\operatorname {I} (X;Y)\\&=\mathrm {H} (X\mid Y)+\mathrm {H} (Y\mid X)\\&=2\mathrm {H} (X,Y)-\mathrm {H} (X)-\mathrm {H} (Y)\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mi>d</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mn>2</mn> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>∣</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mtd> </mtr> <mtr> <mtd /> <mtd> <mi></mi> <mo>=</mo> <mn>2</mn> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}d(X,Y)&=\mathrm {H} (X,Y)-\operatorname {I} (X;Y)\\&=\mathrm {H} (X)+\mathrm {H} (Y)-2\operatorname {I} (X;Y)\\&=\mathrm {H} (X\mid Y)+\mathrm {H} (Y\mid X)\\&=2\mathrm {H} (X,Y)-\mathrm {H} (X)-\mathrm {H} (Y)\end{aligned}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/08f2e8d516108731c9528d08b9a7617c8d28adfe" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -5.671ex; width:37.703ex; height:12.509ex;" alt="{\displaystyle {\begin{aligned}d(X,Y)&=\mathrm {H} (X,Y)-\operatorname {I} (X;Y)\\&=\mathrm {H} (X)+\mathrm {H} (Y)-2\operatorname {I} (X;Y)\\&=\mathrm {H} (X\mid Y)+\mathrm {H} (Y\mid X)\\&=2\mathrm {H} (X,Y)-\mathrm {H} (X)-\mathrm {H} (Y)\end{aligned}}}"></span></dd></dl> <p>satisfies the properties of a metric (<a href="/wiki/Triangle_inequality" title="Triangle inequality">triangle inequality</a>, <a href="/wiki/Non-negative" class="mw-redirect" title="Non-negative">non-negativity</a>, <a href="/wiki/Identity_of_indiscernibles" title="Identity of indiscernibles">indiscernability</a> and symmetry), where equality <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X=Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> <mo>=</mo> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X=Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f0b549c2e838e7da9dfd27fe4e1eac711331bdef" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.852ex; height:2.176ex;" alt="{\displaystyle X=Y}"></span> is understood to mean that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> can be completely determined from <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>.<sup id="cite_ref-10" class="reference"><a href="#cite_note-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> </p><p>This distance metric is also known as the <a href="/wiki/Variation_of_information" title="Variation of information">variation of information</a>. </p><p>If <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X,Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X,Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b8705438171d938b7f59cd1bfa5b7d99b6afa5cd" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.787ex; height:2.509ex;" alt="{\displaystyle X,Y}"></span> are discrete random variables then all the entropy terms are non-negative, so <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 0\leq d(X,Y)\leq \mathrm {H} (X,Y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mn>0</mn> <mo>≤</mo> <mi>d</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>≤</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle 0\leq d(X,Y)\leq \mathrm {H} (X,Y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/5661f9cda82dfc77bc3efefd150953eb1c4d8cb5" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:23.511ex; height:2.843ex;" alt="{\displaystyle 0\leq d(X,Y)\leq \mathrm {H} (X,Y)}"></span> and one can define a normalized distance </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D(X,Y)={\frac {d(X,Y)}{\mathrm {H} (X,Y)}}\leq 1.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>D</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>d</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>≤</mo> <mn>1.</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D(X,Y)={\frac {d(X,Y)}{\mathrm {H} (X,Y)}}\leq 1.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/60f35cf46582107bc5a97366faad1bee6a7cc328" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:25.703ex; height:6.509ex;" alt="{\displaystyle D(X,Y)={\frac {d(X,Y)}{\mathrm {H} (X,Y)}}\leq 1.}"></span></dd></dl> <p>Plugging in the definitions shows that </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D(X,Y)=1-{\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>D</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>1</mn> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D(X,Y)=1-{\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0e09c1fbd651d404dbb5ef314bfd807e4ad8df56" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:25.445ex; height:6.509ex;" alt="{\displaystyle D(X,Y)=1-{\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}.}"></span></dd></dl> <p>This is known as the Rajski Distance.<sup id="cite_ref-11" class="reference"><a href="#cite_note-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> In a set-theoretic interpretation of information (see the figure for <a href="/wiki/Conditional_entropy" title="Conditional entropy">Conditional entropy</a>), this is effectively the <a href="/wiki/Jaccard_index" title="Jaccard index">Jaccard distance</a> between <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span>. </p><p>Finally, </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle D^{\prime }(X,Y)=1-{\frac {\operatorname {I} (X;Y)}{\max \left\{\mathrm {H} (X),\mathrm {H} (Y)\right\}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-variant" mathvariant="normal">′</mi> </mrow> </msup> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>1</mn> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mo movablelimits="true" form="prefix">max</mo> <mrow> <mo>{</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mo>}</mo> </mrow> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle D^{\prime }(X,Y)=1-{\frac {\operatorname {I} (X;Y)}{\max \left\{\mathrm {H} (X),\mathrm {H} (Y)\right\}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/724dc23807ebb3e15831d489fb62f69108e0e494" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:36.073ex; height:6.509ex;" alt="{\displaystyle D^{\prime }(X,Y)=1-{\frac {\operatorname {I} (X;Y)}{\max \left\{\mathrm {H} (X),\mathrm {H} (Y)\right\}}}}"></span></dd></dl> <p>is also a metric. </p> <div class="mw-heading mw-heading3"><h3 id="Conditional_mutual_information">Conditional mutual information</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=15" title="Edit section: Conditional mutual information"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1236090951">.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}</style><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Conditional_mutual_information" title="Conditional mutual information">Conditional mutual information</a></div> <p>Sometimes it is useful to express the mutual information of two random variables conditioned on a third. </p> <div class="equation-box" style="margin: 0 0 0 1.6em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <p><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y|Z)=\mathbb {E} _{Z}[D_{\mathrm {KL} }(P_{(X,Y)|Z}\|P_{X|Z}\otimes P_{Y|Z})]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">[</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">K</mi> <mi mathvariant="normal">L</mi> </mrow> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo>⊗</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y|Z)=\mathbb {E} _{Z}[D_{\mathrm {KL} }(P_{(X,Y)|Z}\|P_{X|Z}\otimes P_{Y|Z})]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3291764cf0532907067deb1d560302d1124f7de5" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:44.556ex; height:3.176ex;" alt="{\displaystyle \operatorname {I} (X;Y|Z)=\mathbb {E} _{Z}[D_{\mathrm {KL} }(P_{(X,Y)|Z}\|P_{X|Z}\otimes P_{Y|Z})]}"></span> </p> </div> <p>For jointly <a href="/wiki/Discrete_random_variable" class="mw-redirect" title="Discrete random variable">discrete random variables</a> this takes the form </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y|Z)=\sum _{z\in {\mathcal {Z}}}\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p_{Z}(z)\,p_{X,Y|Z}(x,y|z)\log \left[{\frac {p_{X,Y|Z}(x,y|z)}{p_{X|Z}\,(x|z)p_{Y|Z}(y|z)}}\right]},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> <mo stretchy="false">)</mo> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>z</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>z</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>[</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mspace width="thinmathspace" /> <mo stretchy="false">(</mo> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>]</mo> </mrow> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y|Z)=\sum _{z\in {\mathcal {Z}}}\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p_{Z}(z)\,p_{X,Y|Z}(x,y|z)\log \left[{\frac {p_{X,Y|Z}(x,y|z)}{p_{X|Z}\,(x|z)p_{Y|Z}(y|z)}}\right]},}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ba63d00f0fc6c57b74ab192657205b9422267fbb" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:69.454ex; height:7.676ex;" alt="{\displaystyle \operatorname {I} (X;Y|Z)=\sum _{z\in {\mathcal {Z}}}\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}{p_{Z}(z)\,p_{X,Y|Z}(x,y|z)\log \left[{\frac {p_{X,Y|Z}(x,y|z)}{p_{X|Z}\,(x|z)p_{Y|Z}(y|z)}}\right]},}"></span></dd></dl> <p>which can be simplified as </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y|Z)=\sum _{z\in {\mathcal {Z}}}\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}p_{X,Y,Z}(x,y,z)\log {\frac {p_{X,Y,Z}(x,y,z)p_{Z}(z)}{p_{X,Z}(x,z)p_{Y,Z}(y,z)}}.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> <mo stretchy="false">)</mo> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>z</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </mrow> </munder> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y|Z)=\sum _{z\in {\mathcal {Z}}}\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}p_{X,Y,Z}(x,y,z)\log {\frac {p_{X,Y,Z}(x,y,z)p_{Z}(z)}{p_{X,Z}(x,z)p_{Y,Z}(y,z)}}.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/87714d3da090d91381c3a5b82c3a25f2b9a13be6" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:62.03ex; height:7.176ex;" alt="{\displaystyle \operatorname {I} (X;Y|Z)=\sum _{z\in {\mathcal {Z}}}\sum _{y\in {\mathcal {Y}}}\sum _{x\in {\mathcal {X}}}p_{X,Y,Z}(x,y,z)\log {\frac {p_{X,Y,Z}(x,y,z)p_{Z}(z)}{p_{X,Z}(x,z)p_{Y,Z}(y,z)}}.}"></span></dd></dl> <p>For jointly <a href="/wiki/Continuous_random_variable" class="mw-redirect" title="Continuous random variable">continuous random variables</a> this takes the form </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y|Z)=\int _{\mathcal {Z}}\int _{\mathcal {Y}}\int _{\mathcal {X}}{p_{Z}(z)\,p_{X,Y|Z}(x,y|z)\log \left[{\frac {p_{X,Y|Z}(x,y|z)}{p_{X|Z}\,(x|z)p_{Y|Z}(y|z)}}\right]}dxdydz,}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </msub> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </msub> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>z</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>[</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mspace width="thinmathspace" /> <mo stretchy="false">(</mo> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>]</mo> </mrow> </mrow> <mi>d</mi> <mi>x</mi> <mi>d</mi> <mi>y</mi> <mi>d</mi> <mi>z</mi> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y|Z)=\int _{\mathcal {Z}}\int _{\mathcal {Y}}\int _{\mathcal {X}}{p_{Z}(z)\,p_{X,Y|Z}(x,y|z)\log \left[{\frac {p_{X,Y|Z}(x,y|z)}{p_{X|Z}\,(x|z)p_{Y|Z}(y|z)}}\right]}dxdydz,}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c398f8106e8dac1bba2a06eabee5230c14cf0a8f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:74.937ex; height:7.509ex;" alt="{\displaystyle \operatorname {I} (X;Y|Z)=\int _{\mathcal {Z}}\int _{\mathcal {Y}}\int _{\mathcal {X}}{p_{Z}(z)\,p_{X,Y|Z}(x,y|z)\log \left[{\frac {p_{X,Y|Z}(x,y|z)}{p_{X|Z}\,(x|z)p_{Y|Z}(y|z)}}\right]}dxdydz,}"></span></dd></dl> <p>which can be simplified as </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y|Z)=\int _{\mathcal {Z}}\int _{\mathcal {Y}}\int _{\mathcal {X}}p_{X,Y,Z}(x,y,z)\log {\frac {p_{X,Y,Z}(x,y,z)p_{Z}(z)}{p_{X,Z}(x,z)p_{Y,Z}(y,z)}}dxdydz.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Z</mi> </mrow> </mrow> </msub> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">Y</mi> </mrow> </mrow> </msub> <msub> <mo>∫</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">X</mi> </mrow> </mrow> </msub> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo>,</mo> <mi>z</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mi>d</mi> <mi>x</mi> <mi>d</mi> <mi>y</mi> <mi>d</mi> <mi>z</mi> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y|Z)=\int _{\mathcal {Z}}\int _{\mathcal {Y}}\int _{\mathcal {X}}p_{X,Y,Z}(x,y,z)\log {\frac {p_{X,Y,Z}(x,y,z)p_{Z}(z)}{p_{X,Z}(x,z)p_{Y,Z}(y,z)}}dxdydz.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2b0de482278bea0c7ffb09f87a4438290bd1a673" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:67.513ex; height:6.509ex;" alt="{\displaystyle \operatorname {I} (X;Y|Z)=\int _{\mathcal {Z}}\int _{\mathcal {Y}}\int _{\mathcal {X}}p_{X,Y,Z}(x,y,z)\log {\frac {p_{X,Y,Z}(x,y,z)p_{Z}(z)}{p_{X,Z}(x,z)p_{Y,Z}(y,z)}}dxdydz.}"></span></dd></dl> <p>Conditioning on a third random variable may either increase or decrease the mutual information, but it is always true that </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y|Z)\geq 0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi>Z</mi> <mo stretchy="false">)</mo> <mo>≥</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y|Z)\geq 0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ead802874dcef297cfdbb7cb1afb48521a86dc88" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.024ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X;Y|Z)\geq 0}"></span></dd></dl> <p>for discrete, jointly distributed random variables <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X,Y,Z}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo>,</mo> <mi>Z</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X,Y,Z}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bcf4a8b48db1a32d24aabe164b07744069093225" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.502ex; height:2.509ex;" alt="{\displaystyle X,Y,Z}"></span>. This result has been used as a basic building block for proving other <a href="/wiki/Inequalities_in_information_theory" title="Inequalities in information theory">inequalities in information theory</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Interaction_information">Interaction information</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=16" title="Edit section: Interaction information"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Interaction_information" title="Interaction information">Interaction information</a></div> <p>Several generalizations of mutual information to more than two random variables have been proposed, such as <a href="/wiki/Total_correlation" title="Total correlation">total correlation</a> (or multi-information) and <a href="/wiki/Dual_total_correlation" title="Dual total correlation">dual total correlation</a>. The expression and study of multivariate higher-degree mutual information was achieved in two seemingly independent works: McGill (1954)<sup id="cite_ref-12" class="reference"><a href="#cite_note-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> who called these functions "interaction information", and Hu Kuo Ting (1962).<sup id="cite_ref-On_the_Amount_of_Information_13-0" class="reference"><a href="#cite_note-On_the_Amount_of_Information-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> Interaction information is defined for one variable as follows: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X_{1})=\mathrm {H} (X_{1})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X_{1})=\mathrm {H} (X_{1})}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/85994202d0672b0c71d2adf7ac719a67f99d34a1" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.257ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X_{1})=\mathrm {H} (X_{1})}"></span></dd></dl> <p>and for <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n>1,}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>n</mi> <mo>></mo> <mn>1</mn> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n>1,}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/957e317312fd02f34c1d1c8c80bd8484c29fde6c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.302ex; height:2.509ex;" alt="{\displaystyle n>1,}"></span> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X_{1};\,...\,;X_{n})=\operatorname {I} (X_{1};\,...\,;X_{n-1})-\operatorname {I} (X_{1};\,...\,;X_{n-1}\mid X_{n}).}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <mspace width="thinmathspace" /> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mspace width="thinmathspace" /> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <mspace width="thinmathspace" /> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mspace width="thinmathspace" /> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>−</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <mspace width="thinmathspace" /> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mspace width="thinmathspace" /> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>∣</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X_{1};\,...\,;X_{n})=\operatorname {I} (X_{1};\,...\,;X_{n-1})-\operatorname {I} (X_{1};\,...\,;X_{n-1}\mid X_{n}).}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/35a8d65cc54df7c33fc69030ead26506ef19ca35" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:60.009ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} (X_{1};\,...\,;X_{n})=\operatorname {I} (X_{1};\,...\,;X_{n-1})-\operatorname {I} (X_{1};\,...\,;X_{n-1}\mid X_{n}).}"></span></dd></dl> <p>Some authors reverse the order of the terms on the right-hand side of the preceding equation, which changes the sign when the number of random variables is odd. (And in this case, the single-variable expression becomes the negative of the entropy.) Note that </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X_{1};\ldots ;X_{n-1}\mid X_{n})=\mathbb {E} _{X_{n}}[D_{\mathrm {KL} }(P_{(X_{1},\ldots ,X_{n-1})\mid X_{n}}\|P_{X_{1}\mid X_{n}}\otimes \cdots \otimes P_{X_{n-1}\mid X_{n}})].}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <mo>…</mo> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>∣</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="double-struck">E</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mrow> </msub> <mo stretchy="false">[</mo> <msub> <mi>D</mi> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">K</mi> <mi mathvariant="normal">L</mi> </mrow> </mrow> </msub> <mo stretchy="false">(</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <mo>…</mo> <mo>,</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>∣</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mrow> </msub> <mo fence="false" stretchy="false">‖</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>∣</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mrow> </msub> <mo>⊗</mo> <mo>⋯</mo> <mo>⊗</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mrow> </msub> <mo>∣</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mrow> </msub> <mo stretchy="false">)</mo> <mo stretchy="false">]</mo> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X_{1};\ldots ;X_{n-1}\mid X_{n})=\mathbb {E} _{X_{n}}[D_{\mathrm {KL} }(P_{(X_{1},\ldots ,X_{n-1})\mid X_{n}}\|P_{X_{1}\mid X_{n}}\otimes \cdots \otimes P_{X_{n-1}\mid X_{n}})].}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b56fa053a5252fa20c65d8122fd32fa69ff2b0fd" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:76.535ex; height:3.176ex;" alt="{\displaystyle I(X_{1};\ldots ;X_{n-1}\mid X_{n})=\mathbb {E} _{X_{n}}[D_{\mathrm {KL} }(P_{(X_{1},\ldots ,X_{n-1})\mid X_{n}}\|P_{X_{1}\mid X_{n}}\otimes \cdots \otimes P_{X_{n-1}\mid X_{n}})].}"></span></dd></dl> <div class="mw-heading mw-heading4"><h4 id="Multivariate_statistical_independence">Multivariate statistical independence</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=17" title="Edit section: Multivariate statistical independence"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The multivariate mutual information functions generalize the pairwise independence case that states that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{1},X_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X_{1},X_{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d6099d6fb3c34ad5e22fad9c79c40c4ebfee1ba" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.991ex; height:2.509ex;" alt="{\displaystyle X_{1},X_{2}}"></span> if and only if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X_{1};X_{2})=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X_{1};X_{2})=0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a7879f556210f16e57fcdbc287d7b940daadcd0d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.233ex; height:2.843ex;" alt="{\displaystyle I(X_{1};X_{2})=0}"></span>, to arbitrary numerous variable. n variables are mutually independent if and only if the <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 2^{n}-n-1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mn>2</mn> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> <mo>−</mo> <mi>n</mi> <mo>−</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle 2^{n}-n-1}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8ba892a0767200f518bf7bf46cd1afbf20c02e3e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:10.619ex; height:2.509ex;" alt="{\displaystyle 2^{n}-n-1}"></span> mutual information functions vanish <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X_{1};\ldots ;X_{k})=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <mo>…</mo> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X_{1};\ldots ;X_{k})=0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ba5ea1c5186057a4fc25bd4ed635dcd72cd7c183" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.412ex; height:2.843ex;" alt="{\displaystyle I(X_{1};\ldots ;X_{k})=0}"></span> with <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n\geq k\geq 2}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>n</mi> <mo>≥</mo> <mi>k</mi> <mo>≥</mo> <mn>2</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n\geq k\geq 2}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/20a64fd1db2ef9118181be539fbee0a2af4d0ccb" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:9.965ex; height:2.343ex;" alt="{\displaystyle n\geq k\geq 2}"></span> (theorem 2<sup id="cite_ref-e21090869_14-0" class="reference"><a href="#cite_note-e21090869-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup>). In this sense, the <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle I(X_{1};\ldots ;X_{k})=0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <mo>…</mo> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo stretchy="false">)</mo> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle I(X_{1};\ldots ;X_{k})=0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ba5ea1c5186057a4fc25bd4ed635dcd72cd7c183" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.412ex; height:2.843ex;" alt="{\displaystyle I(X_{1};\ldots ;X_{k})=0}"></span> can be used as a refined statistical independence criterion. </p> <div class="mw-heading mw-heading4"><h4 id="Applications">Applications</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=18" title="Edit section: Applications"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>For 3 variables, Brenner et al. applied multivariate mutual information to neural coding and called its negativity "synergy"<sup id="cite_ref-15" class="reference"><a href="#cite_note-15"><span class="cite-bracket">[</span>15<span class="cite-bracket">]</span></a></sup> and Watkinson et al. applied it to genetic expression.<sup id="cite_ref-16" class="reference"><a href="#cite_note-16"><span class="cite-bracket">[</span>16<span class="cite-bracket">]</span></a></sup> For arbitrary k variables, Tapia et al. applied multivariate mutual information to gene expression.<sup id="cite_ref-s41598_17-0" class="reference"><a href="#cite_note-s41598-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-e21090869_14-1" class="reference"><a href="#cite_note-e21090869-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup> It can be zero, positive, or negative.<sup id="cite_ref-On_the_Amount_of_Information_13-1" class="reference"><a href="#cite_note-On_the_Amount_of_Information-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> The positivity corresponds to relations generalizing the pairwise correlations, nullity corresponds to a refined notion of independence, and negativity detects high dimensional "emergent" relations and clusterized datapoints <sup id="cite_ref-s41598_17-1" class="reference"><a href="#cite_note-s41598-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup>). </p><p>One high-dimensional generalization scheme which maximizes the mutual information between the joint distribution and other target variables is found to be useful in <a href="/wiki/Feature_selection" title="Feature selection">feature selection</a>.<sup id="cite_ref-18" class="reference"><a href="#cite_note-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup> </p><p>Mutual information is also used in the area of signal processing as a <a href="/wiki/Similarity_measure" title="Similarity measure">measure of similarity</a> between two signals. For example, FMI metric<sup id="cite_ref-19" class="reference"><a href="#cite_note-19"><span class="cite-bracket">[</span>19<span class="cite-bracket">]</span></a></sup> is an image fusion performance measure that makes use of mutual information in order to measure the amount of information that the fused image contains about the source images. The <a href="/wiki/Matlab" class="mw-redirect" title="Matlab">Matlab</a> code for this metric can be found at.<sup id="cite_ref-20" class="reference"><a href="#cite_note-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup> A python package for computing all multivariate mutual informations, conditional mutual information, joint entropies, total correlations, information distance in a dataset of n variables is available.<sup id="cite_ref-21" class="reference"><a href="#cite_note-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Directed_information">Directed information</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=19" title="Edit section: Directed information"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p><a href="/wiki/Directed_information" title="Directed information">Directed information</a>, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} \left(X^{n}\to Y^{n}\right)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <msup> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> <mo stretchy="false">→</mo> <msup> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mrow> <mo>)</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} \left(X^{n}\to Y^{n}\right)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8fae56c94d4c4c42076205d58b6f35b6b7823e6e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:12.597ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} \left(X^{n}\to Y^{n}\right)}"></span>, measures the amount of information that flows from the process <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X^{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/268db8293666fefd75cfb00513706171948edf09" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.215ex; height:2.343ex;" alt="{\displaystyle X^{n}}"></span> to <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y^{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a6e8fe1b832e2088d10ef7c91d815a0d21aad4ca" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:3.119ex; height:2.176ex;" alt="{\displaystyle Y^{n}}"></span>, where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X^{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/268db8293666fefd75cfb00513706171948edf09" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.215ex; height:2.343ex;" alt="{\displaystyle X^{n}}"></span> denotes the vector <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{1},X_{2},...,X_{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mo>,</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X_{1},X_{2},...,X_{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/79889d4b30634a6a4e10aadff7c7f21c122d0b89" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:15.303ex; height:2.509ex;" alt="{\displaystyle X_{1},X_{2},...,X_{n}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y^{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msup> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y^{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a6e8fe1b832e2088d10ef7c91d815a0d21aad4ca" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:3.119ex; height:2.176ex;" alt="{\displaystyle Y^{n}}"></span> denotes <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y_{1},Y_{2},...,Y_{n}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>,</mo> <mo>.</mo> <mo>.</mo> <mo>.</mo> <mo>,</mo> <msub> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y_{1},Y_{2},...,Y_{n}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4d012bcf0600a7ada2dca8e6c31836c16733c1db" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:13.582ex; height:2.509ex;" alt="{\displaystyle Y_{1},Y_{2},...,Y_{n}}"></span>. The term <i>directed information</i> was coined by <a href="/wiki/James_Massey" title="James Massey">James Massey</a> and is defined as </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} \left(X^{n}\to Y^{n}\right)=\sum _{i=1}^{n}\operatorname {I} \left(X^{i};Y_{i}\mid Y^{i-1}\right)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <msup> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> <mo stretchy="false">→</mo> <msup> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </msup> </mrow> <mo>)</mo> </mrow> <mo>=</mo> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>n</mi> </mrow> </munderover> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <msup> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msup> <mo>;</mo> <msub> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>∣</mo> <msup> <mi>Y</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>−</mo> <mn>1</mn> </mrow> </msup> </mrow> <mo>)</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} \left(X^{n}\to Y^{n}\right)=\sum _{i=1}^{n}\operatorname {I} \left(X^{i};Y_{i}\mid Y^{i-1}\right)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ba113fba8d56aaed3dde49921d9501ea235bc30a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:35.125ex; height:6.843ex;" alt="{\displaystyle \operatorname {I} \left(X^{n}\to Y^{n}\right)=\sum _{i=1}^{n}\operatorname {I} \left(X^{i};Y_{i}\mid Y^{i-1}\right)}"></span>.</dd></dl> <p>Note that if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n=1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>n</mi> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n=1}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d9ec7e1edc2e6d98f5aec2a39ae5f1c99d1e1425" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.656ex; height:2.176ex;" alt="{\displaystyle n=1}"></span>, the directed information becomes the mutual information. Directed information has many applications in problems where <a href="/wiki/Causality" title="Causality">causality</a> plays an important role, such as <a href="/wiki/Channel_capacity" title="Channel capacity">capacity of channel</a> with feedback.<sup id="cite_ref-22" class="reference"><a href="#cite_note-22"><span class="cite-bracket">[</span>22<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-23" class="reference"><a href="#cite_note-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Normalized_variants">Normalized variants</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=20" title="Edit section: Normalized variants"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Normalized variants of the mutual information are provided by the <i>coefficients of constraint</i>,<sup id="cite_ref-FOOTNOTECoombsDawesTversky1970_24-0" class="reference"><a href="#cite_note-FOOTNOTECoombsDawesTversky1970-24"><span class="cite-bracket">[</span>24<span class="cite-bracket">]</span></a></sup> <a href="/wiki/Uncertainty_coefficient" title="Uncertainty coefficient">uncertainty coefficient</a><sup id="cite_ref-pressflannery_25-0" class="reference"><a href="#cite_note-pressflannery-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup> or proficiency:<sup id="cite_ref-JimWhite_26-0" class="reference"><a href="#cite_note-JimWhite-26"><span class="cite-bracket">[</span>26<span class="cite-bracket">]</span></a></sup> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle C_{XY}={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (Y)}}~~~~{\mbox{and}}~~~~C_{YX}={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)}}.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>C</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mi>Y</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mtext> </mtext> <mtext> </mtext> <mtext> </mtext> <mtext> </mtext> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mtext>and</mtext> </mstyle> </mrow> <mtext> </mtext> <mtext> </mtext> <mtext> </mtext> <mtext> </mtext> <msub> <mi>C</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mi>X</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle C_{XY}={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (Y)}}~~~~{\mbox{and}}~~~~C_{YX}={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)}}.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d9534b9a73bd56562cd8575d12a08a86bff81b8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:40.877ex; height:6.509ex;" alt="{\displaystyle C_{XY}={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (Y)}}~~~~{\mbox{and}}~~~~C_{YX}={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)}}.}"></span></dd></dl> <p>The two coefficients have a value ranging in [0, 1], but are not necessarily equal. This measure is not symmetric. If one desires a symmetric measure they can consider the following <i><a href="/wiki/Redundancy_(information_theory)" title="Redundancy (information theory)">redundancy</a></i> measure: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle R={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)+\mathrm {H} (Y)}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>R</mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle R={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)+\mathrm {H} (Y)}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/aab0ed087cf87566028f40752cbdc9bd2c6ecb91" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:19.397ex; height:6.509ex;" alt="{\displaystyle R={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)+\mathrm {H} (Y)}}}"></span></dd></dl> <p>which attains a minimum of zero when the variables are independent and a maximum value of </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle R_{\max }={\frac {\min \left\{\mathrm {H} (X),\mathrm {H} (Y)\right\}}{\mathrm {H} (X)+\mathrm {H} (Y)}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>R</mi> <mrow class="MJX-TeXAtom-ORD"> <mo movablelimits="true" form="prefix">max</mo> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mo movablelimits="true" form="prefix">min</mo> <mrow> <mo>{</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mo>}</mo> </mrow> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle R_{\max }={\frac {\min \left\{\mathrm {H} (X),\mathrm {H} (Y)\right\}}{\mathrm {H} (X)+\mathrm {H} (Y)}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fb80f9cf02e548334b770794776dc4c7e8c0e89d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:27.469ex; height:6.509ex;" alt="{\displaystyle R_{\max }={\frac {\min \left\{\mathrm {H} (X),\mathrm {H} (Y)\right\}}{\mathrm {H} (X)+\mathrm {H} (Y)}}}"></span></dd></dl> <p>when one variable becomes completely redundant with the knowledge of the other. See also <i><a href="/wiki/Redundancy_(information_theory)" title="Redundancy (information theory)">Redundancy (information theory)</a></i>. </p><p>Another symmetrical measure is the <i>symmetric uncertainty</i> (<a href="#CITEREFWittenFrank2005">Witten & Frank 2005</a>), given by </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle U(X,Y)=2R=2{\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)+\mathrm {H} (Y)}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>U</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mn>2</mn> <mi>R</mi> <mo>=</mo> <mn>2</mn> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle U(X,Y)=2R=2{\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)+\mathrm {H} (Y)}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c93c8f1b48d915fa7f65b5c855298f3a8a9e6de7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:33.2ex; height:6.509ex;" alt="{\displaystyle U(X,Y)=2R=2{\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X)+\mathrm {H} (Y)}}}"></span></dd></dl> <p>which represents the <a href="/wiki/Harmonic_mean" title="Harmonic mean">harmonic mean</a> of the two uncertainty coefficients <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle C_{XY},C_{YX}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>C</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mi>Y</mi> </mrow> </msub> <mo>,</mo> <msub> <mi>C</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle C_{XY},C_{YX}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/854220ee42ef13238b5526dc5362ad99bd61ac9e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:10.13ex; height:2.509ex;" alt="{\displaystyle C_{XY},C_{YX}}"></span>.<sup id="cite_ref-pressflannery_25-1" class="reference"><a href="#cite_note-pressflannery-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup> </p><p>If we consider mutual information as a special case of the <a href="/wiki/Total_correlation" title="Total correlation">total correlation</a> or <a href="/wiki/Dual_total_correlation" title="Dual total correlation">dual total correlation</a>, the normalized version are respectively, </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\operatorname {I} (X;Y)}{\min \left[\mathrm {H} (X),\mathrm {H} (Y)\right]}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mo movablelimits="true" form="prefix">min</mo> <mrow> <mo>[</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mo>]</mo> </mrow> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\operatorname {I} (X;Y)}{\min \left[\mathrm {H} (X),\mathrm {H} (Y)\right]}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/beff2776011750497c167aa722baea33effa6b6e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:18.284ex; height:6.509ex;" alt="{\displaystyle {\frac {\operatorname {I} (X;Y)}{\min \left[\mathrm {H} (X),\mathrm {H} (Y)\right]}}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}\;.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mspace width="thickmathspace" /> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}\;.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4005a4202a94e9aeae5d3dc5f5c614523a1baebe" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:10.468ex; height:6.509ex;" alt="{\displaystyle {\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}\;.}"></span></dd></dl> <p>This normalized version also known as <b>Information Quality Ratio (IQR)</b> which quantifies the amount of information of a variable based on another variable against total uncertainty:<sup id="cite_ref-DRWijaya_27-0" class="reference"><a href="#cite_note-DRWijaya-27"><span class="cite-bracket">[</span>27<span class="cite-bracket">]</span></a></sup> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle IQR(X,Y)=\operatorname {E} [\operatorname {I} (X;Y)]={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}={\frac {\sum _{x\in X}\sum _{y\in Y}p(x,y)\log {p(x)p(y)}}{\sum _{x\in X}\sum _{y\in Y}p(x,y)\log {p(x,y)}}}-1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>I</mi> <mi>Q</mi> <mi>R</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi mathvariant="normal">E</mi> <mo>⁡</mo> <mo stretchy="false">[</mo> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo stretchy="false">]</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo>,</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mi>X</mi> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mi>Y</mi> </mrow> </munder> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mi>p</mi> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mrow> <mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mi>X</mi> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mi>Y</mi> </mrow> </munder> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mrow> </mfrac> </mrow> <mo>−</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle IQR(X,Y)=\operatorname {E} [\operatorname {I} (X;Y)]={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}={\frac {\sum _{x\in X}\sum _{y\in Y}p(x,y)\log {p(x)p(y)}}{\sum _{x\in X}\sum _{y\in Y}p(x,y)\log {p(x,y)}}}-1}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6392ff4132d19d05ee2740cb15dee7abffa3df0c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:76.43ex; height:7.176ex;" alt="{\displaystyle IQR(X,Y)=\operatorname {E} [\operatorname {I} (X;Y)]={\frac {\operatorname {I} (X;Y)}{\mathrm {H} (X,Y)}}={\frac {\sum _{x\in X}\sum _{y\in Y}p(x,y)\log {p(x)p(y)}}{\sum _{x\in X}\sum _{y\in Y}p(x,y)\log {p(x,y)}}}-1}"></span></dd></dl> <p>There's a normalization<sup id="cite_ref-strehl-jmlr02_28-0" class="reference"><a href="#cite_note-strehl-jmlr02-28"><span class="cite-bracket">[</span>28<span class="cite-bracket">]</span></a></sup> which derives from first thinking of mutual information as an analogue to <a href="/wiki/Covariance" title="Covariance">covariance</a> (thus <a href="/wiki/Entropy_(information_theory)" title="Entropy (information theory)">Shannon entropy</a> is analogous to <a href="/wiki/Variance" title="Variance">variance</a>). Then the normalized mutual information is calculated akin to the <a href="/wiki/Pearson_product-moment_correlation_coefficient" class="mw-redirect" title="Pearson product-moment correlation coefficient">Pearson correlation coefficient</a>, </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\operatorname {I} (X;Y)}{\sqrt {\mathrm {H} (X)\mathrm {H} (Y)}}}\;.}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </mrow> <msqrt> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <mi>Y</mi> <mo stretchy="false">)</mo> </msqrt> </mfrac> </mrow> <mspace width="thickmathspace" /> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\frac {\operatorname {I} (X;Y)}{\sqrt {\mathrm {H} (X)\mathrm {H} (Y)}}}\;.}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8fc8133690f32ab83dbfba5abda74d726938989f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:15.31ex; height:7.009ex;" alt="{\displaystyle {\frac {\operatorname {I} (X;Y)}{\sqrt {\mathrm {H} (X)\mathrm {H} (Y)}}}\;.}"></span></dd></dl> <div class="mw-heading mw-heading3"><h3 id="Weighted_variants">Weighted variants</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=21" title="Edit section: Weighted variants"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>In the traditional formulation of the mutual information, </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in Y}\sum _{x\in X}p(x,y)\log {\frac {p(x,y)}{p(x)\,p(y)}},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mi>Y</mi> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mi>X</mi> </mrow> </munder> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <mi>p</mi> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in Y}\sum _{x\in X}p(x,y)\log {\frac {p(x,y)}{p(x)\,p(y)}},}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6301125f185b77f922dd620eda558349d8d0f6ec" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:38.657ex; height:7.176ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in Y}\sum _{x\in X}p(x,y)\log {\frac {p(x,y)}{p(x)\,p(y)}},}"></span></dd></dl> <p>each <i>event</i> or <i>object</i> specified by <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/41cf50e4a314ca8e2c30964baa8d26e5be7a9386" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.328ex; height:2.843ex;" alt="{\displaystyle (x,y)}"></span> is weighted by the corresponding probability <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089e91a1824e14cebc8e8d04dc652c61b3008e0a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:6.587ex; height:2.843ex;" alt="{\displaystyle p(x,y)}"></span>. This assumes that all objects or events are equivalent <i>apart from</i> their probability of occurrence. However, in some applications it may be the case that certain objects or events are more <i>significant</i> than others, or that certain patterns of association are more semantically important than others. </p><p>For example, the deterministic mapping <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(1,1),(2,2),(3,3)\}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">{</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>2</mn> <mo>,</mo> <mn>2</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>3</mn> <mo>,</mo> <mn>3</mn> <mo stretchy="false">)</mo> <mo fence="false" stretchy="false">}</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \{(1,1),(2,2),(3,3)\}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8b13cd0103f9e43410e877754765a7ad994fe3cc" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.897ex; height:2.843ex;" alt="{\displaystyle \{(1,1),(2,2),(3,3)\}}"></span> may be viewed as stronger than the deterministic mapping <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(1,3),(2,1),(3,2)\}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">{</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>,</mo> <mn>3</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>2</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>3</mn> <mo>,</mo> <mn>2</mn> <mo stretchy="false">)</mo> <mo fence="false" stretchy="false">}</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \{(1,3),(2,1),(3,2)\}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/121771fac14b709e1be2dc4d9c3e4a04a0dd9715" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.897ex; height:2.843ex;" alt="{\displaystyle \{(1,3),(2,1),(3,2)\}}"></span>, although these relationships would yield the same mutual information. This is because the mutual information is not sensitive at all to any inherent ordering in the variable values (<a href="#CITEREFCronbach1954">Cronbach 1954</a>, <a href="#CITEREFCoombsDawesTversky1970">Coombs, Dawes & Tversky 1970</a>, <a href="#CITEREFLockhead1970">Lockhead 1970</a>), and is therefore not sensitive at all to the <b>form</b> of the relational mapping between the associated variables. If it is desired that the former relation—showing agreement on all variable values—be judged stronger than the later relation, then it is possible to use the following <i>weighted mutual information</i> (<a href="#CITEREFGuiasu1977">Guiasu 1977</a>). </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in Y}\sum _{x\in X}w(x,y)p(x,y)\log {\frac {p(x,y)}{p(x)\,p(y)}},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>y</mi> <mo>∈</mo> <mi>Y</mi> </mrow> </munder> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>x</mi> <mo>∈</mo> <mi>X</mi> </mrow> </munder> <mi>w</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <mspace width="thinmathspace" /> <mi>p</mi> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in Y}\sum _{x\in X}w(x,y)p(x,y)\log {\frac {p(x,y)}{p(x)\,p(y)}},}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d2819877e9ba67b97cc5a777f2f18c72de62d87e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:45.649ex; height:7.176ex;" alt="{\displaystyle \operatorname {I} (X;Y)=\sum _{y\in Y}\sum _{x\in X}w(x,y)p(x,y)\log {\frac {p(x,y)}{p(x)\,p(y)}},}"></span></dd></dl> <p>which places a weight <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2c832b255202f489718326a29d60b7c2be485b15" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.993ex; height:2.843ex;" alt="{\displaystyle w(x,y)}"></span> on the probability of each variable value co-occurrence, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p(x,y)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p(x,y)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/089e91a1824e14cebc8e8d04dc652c61b3008e0a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; margin-left: -0.089ex; width:6.587ex; height:2.843ex;" alt="{\displaystyle p(x,y)}"></span>. This allows that certain probabilities may carry more or less significance than others, thereby allowing the quantification of relevant <i>holistic</i> or <i><a href="/wiki/Pr%C3%A4gnanz" class="mw-redirect" title="Prägnanz">Prägnanz</a></i> factors. In the above example, using larger relative weights for <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w(1,1)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <mo stretchy="false">(</mo> <mn>1</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w(1,1)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e01c54cc04db0d799c7e7f2407adcee0d6e5b6e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.832ex; height:2.843ex;" alt="{\displaystyle w(1,1)}"></span>, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w(2,2)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <mo stretchy="false">(</mo> <mn>2</mn> <mo>,</mo> <mn>2</mn> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w(2,2)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7c748432c52030777e6e130bcf2caa62d3efcab3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.832ex; height:2.843ex;" alt="{\displaystyle w(2,2)}"></span>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w(3,3)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>w</mi> <mo stretchy="false">(</mo> <mn>3</mn> <mo>,</mo> <mn>3</mn> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w(3,3)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/80b9ac172050e00d5cd80e25baceb427b05413d7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.832ex; height:2.843ex;" alt="{\displaystyle w(3,3)}"></span> would have the effect of assessing greater <i>informativeness</i> for the relation <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(1,1),(2,2),(3,3)\}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">{</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>2</mn> <mo>,</mo> <mn>2</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>3</mn> <mo>,</mo> <mn>3</mn> <mo stretchy="false">)</mo> <mo fence="false" stretchy="false">}</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \{(1,1),(2,2),(3,3)\}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8b13cd0103f9e43410e877754765a7ad994fe3cc" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.897ex; height:2.843ex;" alt="{\displaystyle \{(1,1),(2,2),(3,3)\}}"></span> than for the relation <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(1,3),(2,1),(3,2)\}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo fence="false" stretchy="false">{</mo> <mo stretchy="false">(</mo> <mn>1</mn> <mo>,</mo> <mn>3</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>2</mn> <mo>,</mo> <mn>1</mn> <mo stretchy="false">)</mo> <mo>,</mo> <mo stretchy="false">(</mo> <mn>3</mn> <mo>,</mo> <mn>2</mn> <mo stretchy="false">)</mo> <mo fence="false" stretchy="false">}</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \{(1,3),(2,1),(3,2)\}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/121771fac14b709e1be2dc4d9c3e4a04a0dd9715" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.897ex; height:2.843ex;" alt="{\displaystyle \{(1,3),(2,1),(3,2)\}}"></span>, which may be desirable in some cases of pattern recognition, and the like. This weighted mutual information is a form of weighted KL-Divergence, which is known to take negative values for some inputs,<sup id="cite_ref-weighted-kl_29-0" class="reference"><a href="#cite_note-weighted-kl-29"><span class="cite-bracket">[</span>29<span class="cite-bracket">]</span></a></sup> and there are examples where the weighted mutual information also takes negative values.<sup id="cite_ref-30" class="reference"><a href="#cite_note-30"><span class="cite-bracket">[</span>30<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Adjusted_mutual_information">Adjusted mutual information</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=22" title="Edit section: Adjusted mutual information"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Adjusted_mutual_information" title="Adjusted mutual information">adjusted mutual information</a></div> <p>A probability distribution can be viewed as a <a href="/wiki/Partition_of_a_set" title="Partition of a set">partition of a set</a>. One may then ask: if a set were partitioned randomly, what would the distribution of probabilities be? What would the expectation value of the mutual information be? The <a href="/wiki/Adjusted_mutual_information" title="Adjusted mutual information">adjusted mutual information</a> or AMI subtracts the expectation value of the MI, so that the AMI is zero when two different distributions are random, and one when two distributions are identical. The AMI is defined in analogy to the <a href="/wiki/Adjusted_Rand_index" class="mw-redirect" title="Adjusted Rand index">adjusted Rand index</a> of two different partitions of a set. </p> <div class="mw-heading mw-heading3"><h3 id="Absolute_mutual_information">Absolute mutual information</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=23" title="Edit section: Absolute mutual information"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Using the ideas of <a href="/wiki/Kolmogorov_complexity" title="Kolmogorov complexity">Kolmogorov complexity</a>, one can consider the mutual information of two sequences independent of any probability distribution: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} _{K}(X;Y)=K(X)-K(X\mid Y).}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi mathvariant="normal">I</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>K</mi> </mrow> </msub> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi>K</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo stretchy="false">)</mo> <mo>−</mo> <mi>K</mi> <mo stretchy="false">(</mo> <mi>X</mi> <mo>∣</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>.</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} _{K}(X;Y)=K(X)-K(X\mid Y).}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/729f5b0b7d063f485913b2c997e20a958b288f12" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:31.136ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} _{K}(X;Y)=K(X)-K(X\mid Y).}"></span></dd></dl> <p>To establish that this quantity is symmetric up to a logarithmic factor (<span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} _{K}(X;Y)\approx \operatorname {I} _{K}(Y;X)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi mathvariant="normal">I</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>K</mi> </mrow> </msub> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>X</mi> <mo>;</mo> <mi>Y</mi> <mo stretchy="false">)</mo> <mo>≈</mo> <msub> <mi mathvariant="normal">I</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>K</mi> </mrow> </msub> <mo>⁡</mo> <mo stretchy="false">(</mo> <mi>Y</mi> <mo>;</mo> <mi>X</mi> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} _{K}(X;Y)\approx \operatorname {I} _{K}(Y;X)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4b9abb8b11bcd5e7d1b12c83b488d26d681593ac" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:21.357ex; height:2.843ex;" alt="{\displaystyle \operatorname {I} _{K}(X;Y)\approx \operatorname {I} _{K}(Y;X)}"></span>) one requires the <a href="/wiki/Chain_rule_for_Kolmogorov_complexity" title="Chain rule for Kolmogorov complexity">chain rule for Kolmogorov complexity</a> (<a href="#CITEREFLiVitányi1997">Li & Vitányi 1997</a>). Approximations of this quantity via <a href="/wiki/Data_compression" title="Data compression">compression</a> can be used to define a <a href="/wiki/Metric_(mathematics)" class="mw-redirect" title="Metric (mathematics)">distance measure</a> to perform a <a href="/wiki/Hierarchical_clustering" title="Hierarchical clustering">hierarchical clustering</a> of sequences without having any <a href="/wiki/Domain_knowledge" title="Domain knowledge">domain knowledge</a> of the sequences (<a href="#CITEREFCilibrasiVitányi2005">Cilibrasi & Vitányi 2005</a>). </p> <div class="mw-heading mw-heading3"><h3 id="Linear_correlation">Linear correlation</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=24" title="Edit section: Linear correlation"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Unlike correlation coefficients, such as the <a href="/wiki/Product_moment_correlation_coefficient" class="mw-redirect" title="Product moment correlation coefficient">product moment correlation coefficient</a>, mutual information contains information about all dependence—linear and nonlinear—and not just linear dependence as the correlation coefficient measures. However, in the narrow case that the joint distribution for <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> is a <a href="/wiki/Bivariate_normal_distribution" class="mw-redirect" title="Bivariate normal distribution">bivariate normal distribution</a> (implying in particular that both marginal distributions are normally distributed), there is an exact relationship between <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b4b00e3aab9a75a11379d3a7fd226fc865a26d63" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.84ex; height:2.176ex;" alt="{\displaystyle \operatorname {I} }"></span> and the correlation coefficient <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \rho }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>ρ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \rho }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1f7d439671d1289b6a816e6af7a304be40608d64" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.202ex; height:2.176ex;" alt="{\displaystyle \rho }"></span> (<a href="#CITEREFGel'fandYaglom1957">Gel'fand & Yaglom 1957</a>). </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} =-{\frac {1}{2}}\log \left(1-\rho ^{2}\right)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>=</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <mn>1</mn> <mo>−</mo> <msup> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mrow> <mo>)</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} =-{\frac {1}{2}}\log \left(1-\rho ^{2}\right)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/147f0f4c3ec8f7c47f3d52e9049b6a5e514bc508" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:19.492ex; height:5.176ex;" alt="{\displaystyle \operatorname {I} =-{\frac {1}{2}}\log \left(1-\rho ^{2}\right)}"></span></dd></dl> <p>The equation above can be derived as follows for a bivariate Gaussian: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}{\begin{pmatrix}X_{1}\\X_{2}\end{pmatrix}}&\sim {\mathcal {N}}\left({\begin{pmatrix}\mu _{1}\\\mu _{2}\end{pmatrix}},\Sigma \right),\qquad \Sigma ={\begin{pmatrix}\sigma _{1}^{2}&\rho \sigma _{1}\sigma _{2}\\\rho \sigma _{1}\sigma _{2}&\sigma _{2}^{2}\end{pmatrix}}\\\mathrm {H} (X_{i})&={\frac {1}{2}}\log \left(2\pi e\sigma _{i}^{2}\right)={\frac {1}{2}}+{\frac {1}{2}}\log(2\pi )+\log \left(\sigma _{i}\right),\quad i\in \{1,2\}\\\mathrm {H} (X_{1},X_{2})&={\frac {1}{2}}\log \left[(2\pi e)^{2}|\Sigma |\right]=1+\log(2\pi )+\log \left(\sigma _{1}\sigma _{2}\right)+{\frac {1}{2}}\log \left(1-\rho ^{2}\right)\\\end{aligned}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mtable columnalign="right left right left right left right left right left right left" rowspacing="3pt" columnspacing="0em 2em 0em 2em 0em 2em 0em 2em 0em 2em 0em" displaystyle="true"> <mtr> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>(</mo> <mtable rowspacing="4pt" columnspacing="1em"> <mtr> <mtd> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mtd> </mtr> </mtable> <mo>)</mo> </mrow> </mrow> </mtd> <mtd> <mi></mi> <mo>∼</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi class="MJX-tex-caligraphic" mathvariant="script">N</mi> </mrow> </mrow> <mrow> <mo>(</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>(</mo> <mtable rowspacing="4pt" columnspacing="1em"> <mtr> <mtd> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd> <msub> <mi>μ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mtd> </mtr> </mtable> <mo>)</mo> </mrow> </mrow> <mo>,</mo> <mi mathvariant="normal">Σ</mi> </mrow> <mo>)</mo> </mrow> <mo>,</mo> <mspace width="2em" /> <mi mathvariant="normal">Σ</mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>(</mo> <mtable rowspacing="4pt" columnspacing="1em"> <mtr> <mtd> <msubsup> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mtd> <mtd> <mi>ρ</mi> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mtd> </mtr> <mtr> <mtd> <mi>ρ</mi> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mtd> <mtd> <msubsup> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mtd> </mtr> </mtable> <mo>)</mo> </mrow> </mrow> </mtd> </mtr> <mtr> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <mn>2</mn> <mi>π</mi> <mi>e</mi> <msubsup> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msubsup> </mrow> <mo>)</mo> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mi>log</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mn>2</mn> <mi>π</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> </mrow> </msub> <mo>)</mo> </mrow> <mo>,</mo> <mspace width="1em" /> <mi>i</mi> <mo>∈</mo> <mo fence="false" stretchy="false">{</mo> <mn>1</mn> <mo>,</mo> <mn>2</mn> <mo fence="false" stretchy="false">}</mo> </mtd> </mtr> <mtr> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mo stretchy="false">(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo stretchy="false">)</mo> </mtd> <mtd> <mi></mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>[</mo> <mrow> <mo stretchy="false">(</mo> <mn>2</mn> <mi>π</mi> <mi>e</mi> <msup> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <mi mathvariant="normal">Σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> </mrow> <mo>]</mo> </mrow> <mo>=</mo> <mn>1</mn> <mo>+</mo> <mi>log</mi> <mo>⁡</mo> <mo stretchy="false">(</mo> <mn>2</mn> <mi>π</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <msub> <mi>σ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mo>)</mo> </mrow> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <mn>1</mn> <mo>−</mo> <msup> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mrow> <mo>)</mo> </mrow> </mtd> </mtr> </mtable> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}{\begin{pmatrix}X_{1}\\X_{2}\end{pmatrix}}&\sim {\mathcal {N}}\left({\begin{pmatrix}\mu _{1}\\\mu _{2}\end{pmatrix}},\Sigma \right),\qquad \Sigma ={\begin{pmatrix}\sigma _{1}^{2}&\rho \sigma _{1}\sigma _{2}\\\rho \sigma _{1}\sigma _{2}&\sigma _{2}^{2}\end{pmatrix}}\\\mathrm {H} (X_{i})&={\frac {1}{2}}\log \left(2\pi e\sigma _{i}^{2}\right)={\frac {1}{2}}+{\frac {1}{2}}\log(2\pi )+\log \left(\sigma _{i}\right),\quad i\in \{1,2\}\\\mathrm {H} (X_{1},X_{2})&={\frac {1}{2}}\log \left[(2\pi e)^{2}|\Sigma |\right]=1+\log(2\pi )+\log \left(\sigma _{1}\sigma _{2}\right)+{\frac {1}{2}}\log \left(1-\rho ^{2}\right)\\\end{aligned}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/548ed20f24c64370dcf1fc5c9b7067a2510b8e9a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -8.171ex; width:74.451ex; height:17.509ex;" alt="{\displaystyle {\begin{aligned}{\begin{pmatrix}X_{1}\\X_{2}\end{pmatrix}}&\sim {\mathcal {N}}\left({\begin{pmatrix}\mu _{1}\\\mu _{2}\end{pmatrix}},\Sigma \right),\qquad \Sigma ={\begin{pmatrix}\sigma _{1}^{2}&\rho \sigma _{1}\sigma _{2}\\\rho \sigma _{1}\sigma _{2}&\sigma _{2}^{2}\end{pmatrix}}\\\mathrm {H} (X_{i})&={\frac {1}{2}}\log \left(2\pi e\sigma _{i}^{2}\right)={\frac {1}{2}}+{\frac {1}{2}}\log(2\pi )+\log \left(\sigma _{i}\right),\quad i\in \{1,2\}\\\mathrm {H} (X_{1},X_{2})&={\frac {1}{2}}\log \left[(2\pi e)^{2}|\Sigma |\right]=1+\log(2\pi )+\log \left(\sigma _{1}\sigma _{2}\right)+{\frac {1}{2}}\log \left(1-\rho ^{2}\right)\\\end{aligned}}}"></span></dd></dl> <p>Therefore, </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {I} \left(X_{1};X_{2}\right)=\mathrm {H} \left(X_{1}\right)+\mathrm {H} \left(X_{2}\right)-\mathrm {H} \left(X_{1},X_{2}\right)=-{\frac {1}{2}}\log \left(1-\rho ^{2}\right)}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi mathvariant="normal">I</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>;</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mo>)</mo> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mrow> <mo>(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>)</mo> </mrow> <mo>+</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mrow> <mo>(</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> <mo>)</mo> </mrow> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">H</mi> </mrow> <mrow> <mo>(</mo> <mrow> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>,</mo> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mrow> <mo>)</mo> </mrow> <mo>=</mo> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <mn>2</mn> </mfrac> </mrow> <mi>log</mi> <mo>⁡</mo> <mrow> <mo>(</mo> <mrow> <mn>1</mn> <mo>−</mo> <msup> <mi>ρ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mrow> <mo>)</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \operatorname {I} \left(X_{1};X_{2}\right)=\mathrm {H} \left(X_{1}\right)+\mathrm {H} \left(X_{2}\right)-\mathrm {H} \left(X_{1},X_{2}\right)=-{\frac {1}{2}}\log \left(1-\rho ^{2}\right)}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c37f8dd27680741bee7488e57dfc96db7884508b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:61.838ex; height:5.176ex;" alt="{\displaystyle \operatorname {I} \left(X_{1};X_{2}\right)=\mathrm {H} \left(X_{1}\right)+\mathrm {H} \left(X_{2}\right)-\mathrm {H} \left(X_{1},X_{2}\right)=-{\frac {1}{2}}\log \left(1-\rho ^{2}\right)}"></span></dd></dl> <div class="mw-heading mw-heading3"><h3 id="For_discrete_data">For discrete data</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=25" title="Edit section: For discrete data"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>When <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> are limited to be in a discrete number of states, observation data is summarized in a <a href="/wiki/Contingency_table" title="Contingency table">contingency table</a>, with row variable <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>X</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/68baa052181f707c662844a465bfeeb135e82bab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}"></span> (or <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>i</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle i}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/add78d8608ad86e54951b8c8bd6c8d8416533d20" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}"></span>) and column variable <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>Y</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Y}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/961d67d6b454b4df2301ac571808a3538b3a6d3f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.171ex; width:1.773ex; height:2.009ex;" alt="{\displaystyle Y}"></span> (or <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle j}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>j</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle j}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2f461e54f5c093e92a55547b9764291390f0b5d0" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.027ex; width:0.985ex; height:2.509ex;" alt="{\displaystyle j}"></span>). Mutual information is one of the measures of <a href="/wiki/Association_(statistics)" class="mw-redirect" title="Association (statistics)">association</a> or <a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">correlation</a> between the row and column variables. </p><p>Other measures of association include <a href="/wiki/Pearson%27s_chi-squared_test" title="Pearson's chi-squared test">Pearson's chi-squared test</a> statistics, <a href="/wiki/G-test" title="G-test">G-test</a> statistics, etc. In fact, with the same log base, mutual information will be equal to the <a href="/wiki/G-test" title="G-test">G-test</a> log-likelihood statistic divided by <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 2N}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mn>2</mn> <mi>N</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle 2N}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/eacbd5b0e609e1f3d7da751ac0d50113d27d22aa" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.226ex; height:2.176ex;" alt="{\displaystyle 2N}"></span>, where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>N</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle N}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f5e3890c981ae85503089652feb48b191b57aae3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.064ex; height:2.176ex;" alt="{\displaystyle N}"></span> is the sample size. </p> <div class="mw-heading mw-heading2"><h2 id="Applications_2">Applications</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=26" title="Edit section: Applications"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>In many applications, one wants to maximize mutual information (thus increasing dependencies), which is often equivalent to minimizing <a href="/wiki/Conditional_entropy" title="Conditional entropy">conditional entropy</a>. Examples include: </p> <ul><li>In <a href="/wiki/Search_engine_technology" class="mw-redirect" title="Search engine technology">search engine technology</a>, mutual information between phrases and contexts is used as a feature for <a href="/wiki/K-means_clustering" title="K-means clustering">k-means clustering</a> to discover semantic clusters (concepts).<sup id="cite_ref-magerman_31-0" class="reference"><a href="#cite_note-magerman-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup> For example, the mutual information of a bigram might be calculated as:</li></ul> <div class="equation-box" style="margin: 0 0 0 3.2em;padding: 5px; border-width:2px; border-style: solid; border-color: #0073CF; color: inherit;text-align: center; display: table"> <p><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle MI(x,y)=\log {\frac {P_{X,Y}(x,y)}{P_{X}(x)P_{Y}(y)}}\approx \log {\frac {\frac {f_{XY}}{B}}{{\frac {f_{X}}{U}}{\frac {f_{Y}}{U}}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>M</mi> <mi>I</mi> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mo>,</mo> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo>,</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>x</mi> <mo stretchy="false">)</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>y</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> <mo>≈</mo> <mi>log</mi> <mo>⁡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mfrac> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mi>Y</mi> </mrow> </msub> <mi>B</mi> </mfrac> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> <mi>U</mi> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>Y</mi> </mrow> </msub> <mi>U</mi> </mfrac> </mrow> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle MI(x,y)=\log {\frac {P_{X,Y}(x,y)}{P_{X}(x)P_{Y}(y)}}\approx \log {\frac {\frac {f_{XY}}{B}}{{\frac {f_{X}}{U}}{\frac {f_{Y}}{U}}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/04407b4c31d779f834a474977dfb77eee6f540d7" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -4.005ex; width:41.503ex; height:9.009ex;" alt="{\displaystyle MI(x,y)=\log {\frac {P_{X,Y}(x,y)}{P_{X}(x)P_{Y}(y)}}\approx \log {\frac {\frac {f_{XY}}{B}}{{\frac {f_{X}}{U}}{\frac {f_{Y}}{U}}}}}"></span> </p> </div> <dl><dd>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{XY}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> <mi>Y</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{XY}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ae001a335e2c4b18bd8d91f3956a5086cd01ee5c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.025ex; height:2.509ex;" alt="{\displaystyle f_{XY}}"></span> is the number of times the bigram xy appears in the corpus, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{X}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>f</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>X</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle f_{X}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/17fd6605a04f97c6bedb0a9632f9f023cb18dd40" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.772ex; height:2.509ex;" alt="{\displaystyle f_{X}}"></span> is the number of times the unigram x appears in the corpus, B is the total number of bigrams, and U is the total number of unigrams.<sup id="cite_ref-magerman_31-1" class="reference"><a href="#cite_note-magerman-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup></dd></dl> <ul><li>In <a href="/wiki/Telecommunications" title="Telecommunications">telecommunications</a>, the <a href="/wiki/Channel_capacity" title="Channel capacity">channel capacity</a> is equal to the mutual information, maximized over all input distributions.</li> <li><a href="/wiki/Discriminative_model" title="Discriminative model">Discriminative training</a> procedures for <a href="/wiki/Hidden_Markov_model" title="Hidden Markov model">hidden Markov models</a> have been proposed based on the <a href="/w/index.php?title=Maximum_mutual_information&action=edit&redlink=1" class="new" title="Maximum mutual information (page does not exist)">maximum mutual information</a> (MMI) criterion.</li> <li><a href="/wiki/Nucleic_acid_secondary_structure" title="Nucleic acid secondary structure">RNA secondary structure</a> prediction from a <a href="/wiki/Multiple_sequence_alignment" title="Multiple sequence alignment">multiple sequence alignment</a>.</li> <li><a href="/wiki/Phylogenetic_profiling" title="Phylogenetic profiling">Phylogenetic profiling</a> prediction from pairwise present and disappearance of functionally link <a href="/wiki/Gene" title="Gene">genes</a>.</li> <li>Mutual information has been used as a criterion for <a href="/wiki/Feature_selection" title="Feature selection">feature selection</a> and feature transformations in <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a>. It can be used to characterize both the relevance and redundancy of variables, such as the <a href="/wiki/Minimum_redundancy_feature_selection" title="Minimum redundancy feature selection">minimum redundancy feature selection</a>.</li> <li>Mutual information is used in determining the similarity of two different <a href="/wiki/Cluster_analysis" title="Cluster analysis">clusterings</a> of a dataset. As such, it provides some advantages over the traditional <a href="/wiki/Rand_index" title="Rand index">Rand index</a>.</li> <li>Mutual information of words is often used as a significance function for the computation of <a href="/wiki/Collocation" title="Collocation">collocations</a> in <a href="/wiki/Corpus_linguistics" title="Corpus linguistics">corpus linguistics</a>. This has the added complexity that no word-instance is an instance to two different words; rather, one counts instances where 2 words occur adjacent or in close proximity; this slightly complicates the calculation, since the expected probability of one word occurring within <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>N</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle N}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f5e3890c981ae85503089652feb48b191b57aae3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.064ex; height:2.176ex;" alt="{\displaystyle N}"></span> words of another, goes up with <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>N</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle N}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f5e3890c981ae85503089652feb48b191b57aae3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.064ex; height:2.176ex;" alt="{\displaystyle N}"></span></li> <li>Mutual information is used in <a href="/wiki/Medical_imaging" title="Medical imaging">medical imaging</a> for <a href="/wiki/Image_registration" title="Image registration">image registration</a>. Given a reference image (for example, a brain scan), and a second image which needs to be put into the same <a href="/wiki/Coordinate_system" title="Coordinate system">coordinate system</a> as the reference image, this image is deformed until the mutual information between it and the reference image is maximized.</li> <li>Detection of <a href="/wiki/Phase_synchronization" title="Phase synchronization">phase synchronization</a> in <a href="/wiki/Time_series" title="Time series">time series</a> analysis.</li> <li>In the <a href="/wiki/Infomax" title="Infomax">infomax</a> method for neural-net and other machine learning, including the infomax-based <a href="/wiki/Independent_component_analysis" title="Independent component analysis">Independent component analysis</a> algorithm</li> <li>Average mutual information in <a href="/wiki/Delay_embedding_theorem" class="mw-redirect" title="Delay embedding theorem">delay embedding theorem</a> is used for determining the <i>embedding delay</i> parameter.</li> <li>Mutual information between <a href="/wiki/Genes" class="mw-redirect" title="Genes">genes</a> in <a href="/wiki/Microarray" title="Microarray">expression microarray</a> data is used by the ARACNE algorithm for reconstruction of <a href="/wiki/Gene_regulatory_network" title="Gene regulatory network">gene networks</a>.</li> <li>In <a href="/wiki/Statistical_mechanics" title="Statistical mechanics">statistical mechanics</a>, <a href="/wiki/Loschmidt%27s_paradox" title="Loschmidt's paradox">Loschmidt's paradox</a> may be expressed in terms of mutual information.<sup id="cite_ref-everett56_32-0" class="reference"><a href="#cite_note-everett56-32"><span class="cite-bracket">[</span>32<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-everett57_33-0" class="reference"><a href="#cite_note-everett57-33"><span class="cite-bracket">[</span>33<span class="cite-bracket">]</span></a></sup> Loschmidt noted that it must be impossible to determine a physical law which lacks <a href="/wiki/Time_reversal_symmetry" class="mw-redirect" title="Time reversal symmetry">time reversal symmetry</a> (e.g. the <a href="/wiki/Second_law_of_thermodynamics" title="Second law of thermodynamics">second law of thermodynamics</a>) only from physical laws which have this symmetry. He pointed out that the <a href="/wiki/H-theorem" title="H-theorem">H-theorem</a> of <a href="/wiki/Boltzmann" class="mw-redirect" title="Boltzmann">Boltzmann</a> made the assumption that the velocities of particles in a gas were permanently uncorrelated, which removed the time symmetry inherent in the H-theorem. It can be shown that if a system is described by a probability density in <a href="/wiki/Phase_space" title="Phase space">phase space</a>, then <a href="/wiki/Liouville%27s_theorem_(Hamiltonian)" title="Liouville's theorem (Hamiltonian)">Liouville's theorem</a> implies that the joint information (negative of the joint entropy) of the distribution remains constant in time. The joint information is equal to the mutual information plus the sum of all the marginal information (negative of the marginal entropies) for each particle coordinate. Boltzmann's assumption amounts to ignoring the mutual information in the calculation of entropy, which yields the thermodynamic entropy (divided by the Boltzmann constant).</li> <li>In <a href="/wiki/Stochastic_process" title="Stochastic process">stochastic processes</a> coupled to changing environments, mutual information can be used to disentangle internal and effective environmental dependencies.<sup id="cite_ref-34" class="reference"><a href="#cite_note-34"><span class="cite-bracket">[</span>34<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-35" class="reference"><a href="#cite_note-35"><span class="cite-bracket">[</span>35<span class="cite-bracket">]</span></a></sup> This is particularly useful when a physical system undergoes changes in the parameters describing its dynamics, e.g., changes in temperature.</li> <li>The mutual information is used to learn the structure of <a href="/wiki/Bayesian_network" title="Bayesian network">Bayesian networks</a>/<a href="/wiki/Dynamic_Bayesian_network" title="Dynamic Bayesian network">dynamic Bayesian networks</a>, which is thought to explain the causal relationship between random variables, as exemplified by the GlobalMIT toolkit:<sup id="cite_ref-36" class="reference"><a href="#cite_note-36"><span class="cite-bracket">[</span>36<span class="cite-bracket">]</span></a></sup> learning the globally optimal dynamic Bayesian network with the Mutual Information Test criterion.</li> <li>The mutual information is used to quantify information transmitted during the updating procedure in the <a href="/wiki/Gibbs_sampling" title="Gibbs sampling">Gibbs sampling</a> algorithm.<sup id="cite_ref-37" class="reference"><a href="#cite_note-37"><span class="cite-bracket">[</span>37<span class="cite-bracket">]</span></a></sup></li> <li>Popular cost function in <a href="/wiki/Decision_tree_learning" title="Decision tree learning">decision tree learning</a>.</li> <li>The mutual information is used in <a href="/wiki/Cosmology" title="Cosmology">cosmology</a> to test the influence of large-scale environments on galaxy properties in the <a href="/wiki/Galaxy_Zoo" title="Galaxy Zoo">Galaxy Zoo</a>.</li> <li>The mutual information was used in <a href="/wiki/Solar_Physics" class="mw-redirect" title="Solar Physics">Solar Physics</a> to derive the solar <a href="/wiki/Differential_rotation" title="Differential rotation">differential rotation</a> profile, a travel-time deviation map for sunspots, and a time–distance diagram from quiet-Sun measurements<sup id="cite_ref-38" class="reference"><a href="#cite_note-38"><span class="cite-bracket">[</span>38<span class="cite-bracket">]</span></a></sup></li> <li>Used in Invariant Information Clustering to automatically train neural network classifiers and image segmenters given no labelled data.<sup id="cite_ref-iic_39-0" class="reference"><a href="#cite_note-iic-39"><span class="cite-bracket">[</span>39<span class="cite-bracket">]</span></a></sup></li> <li>In <a href="/wiki/Multiscale_modeling" title="Multiscale modeling">stochastic dynamical systems with multiple timescales</a>, mutual information has been shown to capture the functional couplings between different temporal scales.<sup id="cite_ref-40" class="reference"><a href="#cite_note-40"><span class="cite-bracket">[</span>40<span class="cite-bracket">]</span></a></sup> Importantly, it was shown that physical interactions may or may not give rise to mutual information, depending on the typical timescale of their dynamics.</li></ul> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=27" title="Edit section: See also"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Data_differencing" title="Data differencing">Data differencing</a></li> <li><a href="/wiki/Pointwise_mutual_information" title="Pointwise mutual information">Pointwise mutual information</a></li> <li><a href="/wiki/Quantum_mutual_information" title="Quantum mutual information">Quantum mutual information</a></li> <li><a href="/wiki/Specific-information" title="Specific-information">Specific-information</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="Notes">Notes</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=28" title="Edit section: Notes"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist"> <div class="mw-references-wrap mw-references-columns"><ol class="references"> <li id="cite_note-1"><span class="mw-cite-backlink"><b><a href="#cite_ref-1">^</a></b></span> <span class="reference-text"><style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite id="CITEREFCoverThomas2005" class="citation book cs1">Cover, Thomas M.; Thomas, Joy A. (2005). <a rel="nofollow" class="external text" href="http://www.cs.columbia.edu/~vh/courses/LexicalSemantics/Association/Cover&Thomas-Ch2.pdf"><i>Elements of information theory</i></a> <span class="cs1-format">(PDF)</span>. John Wiley & Sons, Ltd. pp. <span class="nowrap">13–</span>55. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9780471748823" title="Special:BookSources/9780471748823"><bdi>9780471748823</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Elements+of+information+theory&rft.pages=%3Cspan+class%3D%22nowrap%22%3E13-%3C%2Fspan%3E55&rft.pub=John+Wiley+%26+Sons%2C+Ltd&rft.date=2005&rft.isbn=9780471748823&rft.aulast=Cover&rft.aufirst=Thomas+M.&rft.au=Thomas%2C+Joy+A.&rft_id=http%3A%2F%2Fwww.cs.columbia.edu%2F~vh%2Fcourses%2FLexicalSemantics%2FAssociation%2FCover%26Thomas-Ch2.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-2"><span class="mw-cite-backlink"><b><a href="#cite_ref-2">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKreer1957" class="citation journal cs1">Kreer, J. G. (1957). "A question of terminology". <i>IRE Transactions on Information Theory</i>. <b>3</b> (3): 208. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTIT.1957.1057418">10.1109/TIT.1957.1057418</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IRE+Transactions+on+Information+Theory&rft.atitle=A+question+of+terminology&rft.volume=3&rft.issue=3&rft.pages=208&rft.date=1957&rft_id=info%3Adoi%2F10.1109%2FTIT.1957.1057418&rft.aulast=Kreer&rft.aufirst=J.+G.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-cover1991-3"><span class="mw-cite-backlink">^ <a href="#cite_ref-cover1991_3-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-cover1991_3-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-cover1991_3-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCoverThomas1991" class="citation book cs1">Cover, T.M.; Thomas, J.A. (1991). <span class="id-lock-registration" title="Free registration required"><a rel="nofollow" class="external text" href="https://archive.org/details/elementsofinform0000cove"><i>Elements of Information Theory</i></a></span> (Wiley ed.). John Wiley & Sons. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-471-24195-9" title="Special:BookSources/978-0-471-24195-9"><bdi>978-0-471-24195-9</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Elements+of+Information+Theory&rft.edition=Wiley&rft.pub=John+Wiley+%26+Sons&rft.date=1991&rft.isbn=978-0-471-24195-9&rft.aulast=Cover&rft.aufirst=T.M.&rft.au=Thomas%2C+J.A.&rft_id=https%3A%2F%2Farchive.org%2Fdetails%2Felementsofinform0000cove&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-4">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJanssenGuanRobeva2023" class="citation journal cs1">Janssen, Joseph; Guan, Vincent; Robeva, Elina (2023). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v206/janssen23a.html">"Ultra-marginal Feature Importance: Learning from Data with Causal Guarantees"</a>. <i>International Conference on Artificial Intelligence and Statistics</i>: <span class="nowrap">10782–</span>10814. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2204.09938">2204.09938</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=International+Conference+on+Artificial+Intelligence+and+Statistics&rft.atitle=Ultra-marginal+Feature+Importance%3A+Learning+from+Data+with+Causal+Guarantees&rft.pages=%3Cspan+class%3D%22nowrap%22%3E10782-%3C%2Fspan%3E10814&rft.date=2023&rft_id=info%3Aarxiv%2F2204.09938&rft.aulast=Janssen&rft.aufirst=Joseph&rft.au=Guan%2C+Vincent&rft.au=Robeva%2C+Elina&rft_id=https%3A%2F%2Fproceedings.mlr.press%2Fv206%2Fjanssen23a.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-5">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWolpertWolf1995" class="citation journal cs1">Wolpert, D.H.; Wolf, D.R. (1995). "Estimating functions of probability distributions from a finite set of samples". <i>Physical Review E</i>. <b>52</b> (6): <span class="nowrap">6841–</span>6854. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1995PhRvE..52.6841W">1995PhRvE..52.6841W</a>. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.7122">10.1.1.55.7122</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevE.52.6841">10.1103/PhysRevE.52.6841</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/9964199">9964199</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:9795679">9795679</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+E&rft.atitle=Estimating+functions+of+probability+distributions+from+a+finite+set+of+samples&rft.volume=52&rft.issue=6&rft.pages=%3Cspan+class%3D%22nowrap%22%3E6841-%3C%2Fspan%3E6854&rft.date=1995&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A9795679%23id-name%3DS2CID&rft_id=info%3Abibcode%2F1995PhRvE..52.6841W&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.55.7122%23id-name%3DCiteSeerX&rft_id=info%3Apmid%2F9964199&rft_id=info%3Adoi%2F10.1103%2FPhysRevE.52.6841&rft.aulast=Wolpert&rft.aufirst=D.H.&rft.au=Wolf%2C+D.R.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-6">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHutter2001" class="citation journal cs1">Hutter, M. (2001). "Distribution of Mutual Information". <i>Advances in Neural Information Processing Systems</i>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Advances+in+Neural+Information+Processing+Systems&rft.atitle=Distribution+of+Mutual+Information&rft.date=2001&rft.aulast=Hutter&rft.aufirst=M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFArcherParkPillow2013" class="citation journal cs1">Archer, E.; Park, I.M.; Pillow, J. (2013). <a rel="nofollow" class="external text" href="https://doi.org/10.3390%2Fe15051738">"Bayesian and Quasi-Bayesian Estimators for Mutual Information from Discrete Data"</a>. <i>Entropy</i>. <b>15</b> (12): <span class="nowrap">1738–</span>1755. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2013Entrp..15.1738A">2013Entrp..15.1738A</a>. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.294.4690">10.1.1.294.4690</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.3390%2Fe15051738">10.3390/e15051738</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Entropy&rft.atitle=Bayesian+and+Quasi-Bayesian+Estimators+for+Mutual+Information+from+Discrete+Data&rft.volume=15&rft.issue=12&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1738-%3C%2Fspan%3E1755&rft.date=2013&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.294.4690%23id-name%3DCiteSeerX&rft_id=info%3Adoi%2F10.3390%2Fe15051738&rft_id=info%3Abibcode%2F2013Entrp..15.1738A&rft.aulast=Archer&rft.aufirst=E.&rft.au=Park%2C+I.M.&rft.au=Pillow%2C+J.&rft_id=https%3A%2F%2Fdoi.org%2F10.3390%252Fe15051738&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWolpertDeDeo2013" class="citation journal cs1">Wolpert, D.H; DeDeo, S. (2013). <a rel="nofollow" class="external text" href="https://doi.org/10.3390%2Fe15114668">"Estimating Functions of Distributions Defined over Spaces of Unknown Size"</a>. <i>Entropy</i>. <b>15</b> (12): <span class="nowrap">4668–</span>4699. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1311.4548">1311.4548</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2013Entrp..15.4668W">2013Entrp..15.4668W</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.3390%2Fe15114668">10.3390/e15114668</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:2737117">2737117</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Entropy&rft.atitle=Estimating+Functions+of+Distributions+Defined+over+Spaces+of+Unknown+Size&rft.volume=15&rft.issue=12&rft.pages=%3Cspan+class%3D%22nowrap%22%3E4668-%3C%2Fspan%3E4699&rft.date=2013&rft_id=info%3Aarxiv%2F1311.4548&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A2737117%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.3390%2Fe15114668&rft_id=info%3Abibcode%2F2013Entrp..15.4668W&rft.aulast=Wolpert&rft.aufirst=D.H&rft.au=DeDeo%2C+S.&rft_id=https%3A%2F%2Fdoi.org%2F10.3390%252Fe15114668&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFTomasz_JetkaKarol_NienaltowskiTomasz_WinarskiSlawomir_Blonski2019" class="citation cs1">Tomasz Jetka; Karol Nienaltowski; Tomasz Winarski; Slawomir Blonski; Michal Komorowski (2019). "Information-theoretic analysis of multivariate single-cell signaling responses". <i><a href="/wiki/PLOS_Computational_Biology" title="PLOS Computational Biology">PLOS Computational Biology</a></i>. <b>15</b> (7): e1007132. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1808.05581">1808.05581</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2019PLSCB..15E7132J">2019PLSCB..15E7132J</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1371%2Fjournal.pcbi.1007132">10.1371/journal.pcbi.1007132</a></span>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6655862">6655862</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/31299056">31299056</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=PLOS+Computational+Biology&rft.atitle=Information-theoretic+analysis+of+multivariate+single-cell+signaling+responses&rft.volume=15&rft.issue=7&rft.pages=e1007132&rft.date=2019&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6655862%23id-name%3DPMC&rft_id=info%3Abibcode%2F2019PLSCB..15E7132J&rft_id=info%3Aarxiv%2F1808.05581&rft_id=info%3Apmid%2F31299056&rft_id=info%3Adoi%2F10.1371%2Fjournal.pcbi.1007132&rft.au=Tomasz+Jetka&rft.au=Karol+Nienaltowski&rft.au=Tomasz+Winarski&rft.au=Slawomir+Blonski&rft.au=Michal+Komorowski&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-10"><span class="mw-cite-backlink"><b><a href="#cite_ref-10">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRajski1961" class="citation journal cs1">Rajski, C. (1961). "A metric space of discrete probability distributions". <i>Information and Control</i>. <b>4</b> (4): <span class="nowrap">371–</span>377. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0019-9958%2861%2980055-7">10.1016/S0019-9958(61)80055-7</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Information+and+Control&rft.atitle=A+metric+space+of+discrete+probability+distributions&rft.volume=4&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E371-%3C%2Fspan%3E377&rft.date=1961&rft_id=info%3Adoi%2F10.1016%2FS0019-9958%2861%2980055-7&rft.aulast=Rajski&rft.aufirst=C.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-11"><span class="mw-cite-backlink"><b><a href="#cite_ref-11">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRajski1961" class="citation journal cs1">Rajski, C. (1961). "A metric space of discrete probability distributions". <i>Information and Control</i>. <b>4</b> (4): <span class="nowrap">371–</span>377. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0019-9958%2861%2980055-7">10.1016/S0019-9958(61)80055-7</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Information+and+Control&rft.atitle=A+metric+space+of+discrete+probability+distributions&rft.volume=4&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E371-%3C%2Fspan%3E377&rft.date=1961&rft_id=info%3Adoi%2F10.1016%2FS0019-9958%2861%2980055-7&rft.aulast=Rajski&rft.aufirst=C.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-12">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMcGill1954" class="citation journal cs1">McGill, W. (1954). "Multivariate information transmission". <i>Psychometrika</i>. <b>19</b> (1): <span class="nowrap">97–</span>116. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2FBF02289159">10.1007/BF02289159</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:126431489">126431489</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Psychometrika&rft.atitle=Multivariate+information+transmission&rft.volume=19&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E97-%3C%2Fspan%3E116&rft.date=1954&rft_id=info%3Adoi%2F10.1007%2FBF02289159&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A126431489%23id-name%3DS2CID&rft.aulast=McGill&rft.aufirst=W.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-On_the_Amount_of_Information-13"><span class="mw-cite-backlink">^ <a href="#cite_ref-On_the_Amount_of_Information_13-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-On_the_Amount_of_Information_13-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHu1962" class="citation journal cs1">Hu, K.T. (1962). "On the Amount of Information". <i>Theory Probab. Appl</i>. <b>7</b> (4): <span class="nowrap">439–</span>447. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1137%2F1107041">10.1137/1107041</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Theory+Probab.+Appl.&rft.atitle=On+the+Amount+of+Information&rft.volume=7&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E439-%3C%2Fspan%3E447&rft.date=1962&rft_id=info%3Adoi%2F10.1137%2F1107041&rft.aulast=Hu&rft.aufirst=K.T.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-e21090869-14"><span class="mw-cite-backlink">^ <a href="#cite_ref-e21090869_14-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-e21090869_14-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBaudotTapiaBennequinGoaillard2019" class="citation journal cs1">Baudot, P.; Tapia, M.; Bennequin, D.; Goaillard, J.M. (2019). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7515398">"Topological Information Data Analysis"</a>. <i>Entropy</i>. <b>21</b> (9). 869. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1907.04242">1907.04242</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2019Entrp..21..869B">2019Entrp..21..869B</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.3390%2Fe21090869">10.3390/e21090869</a></span>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7515398">7515398</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:195848308">195848308</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Entropy&rft.atitle=Topological+Information+Data+Analysis&rft.volume=21&rft.issue=9&rft.pages=869&rft.date=2019&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC7515398%23id-name%3DPMC&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A195848308%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2019Entrp..21..869B&rft_id=info%3Aarxiv%2F1907.04242&rft_id=info%3Adoi%2F10.3390%2Fe21090869&rft.aulast=Baudot&rft.aufirst=P.&rft.au=Tapia%2C+M.&rft.au=Bennequin%2C+D.&rft.au=Goaillard%2C+J.M.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC7515398&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-15"><span class="mw-cite-backlink"><b><a href="#cite_ref-15">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBrennerStrongKoberleBialek2000" class="citation journal cs1">Brenner, N.; Strong, S.; Koberle, R.; Bialek, W. (2000). "Synergy in a Neural Code". <i>Neural Comput</i>. <b>12</b> (7): <span class="nowrap">1531–</span>1552. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2F089976600300015259">10.1162/089976600300015259</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/10935917">10935917</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:600528">600528</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Neural+Comput&rft.atitle=Synergy+in+a+Neural+Code&rft.volume=12&rft.issue=7&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1531-%3C%2Fspan%3E1552&rft.date=2000&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A600528%23id-name%3DS2CID&rft_id=info%3Apmid%2F10935917&rft_id=info%3Adoi%2F10.1162%2F089976600300015259&rft.aulast=Brenner&rft.aufirst=N.&rft.au=Strong%2C+S.&rft.au=Koberle%2C+R.&rft.au=Bialek%2C+W.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-16"><span class="mw-cite-backlink"><b><a href="#cite_ref-16">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWatkinsonLiangWangZheng2009" class="citation journal cs1">Watkinson, J.; Liang, K.; Wang, X.; Zheng, T.; Anastassiou, D. (2009). "Inference of Regulatory Gene Interactions from Expression Data Using Three-Way Mutual Information". <i>Chall. Syst. Biol. Ann. N. Y. Acad. Sci</i>. <b>1158</b> (1): <span class="nowrap">302–</span>313. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2009NYASA1158..302W">2009NYASA1158..302W</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1111%2Fj.1749-6632.2008.03757.x">10.1111/j.1749-6632.2008.03757.x</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/19348651">19348651</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:8846229">8846229</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Chall.+Syst.+Biol.+Ann.+N.+Y.+Acad.+Sci.&rft.atitle=Inference+of+Regulatory+Gene+Interactions+from+Expression+Data+Using+Three-Way+Mutual+Information&rft.volume=1158&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E302-%3C%2Fspan%3E313&rft.date=2009&rft_id=info%3Adoi%2F10.1111%2Fj.1749-6632.2008.03757.x&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A8846229%23id-name%3DS2CID&rft_id=info%3Apmid%2F19348651&rft_id=info%3Abibcode%2F2009NYASA1158..302W&rft.aulast=Watkinson&rft.aufirst=J.&rft.au=Liang%2C+K.&rft.au=Wang%2C+X.&rft.au=Zheng%2C+T.&rft.au=Anastassiou%2C+D.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-s41598-17"><span class="mw-cite-backlink">^ <a href="#cite_ref-s41598_17-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-s41598_17-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFTapiaBaudotFormizano-TrezinyDufour2018" class="citation journal cs1">Tapia, M.; Baudot, P.; Formizano-Treziny, C.; Dufour, M.; Goaillard, J.M. (2018). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6134142">"Neurotransmitter identity and electrophysiological phenotype are genetically coupled in midbrain dopaminergic neurons"</a>. <i>Sci. Rep</i>. <b>8</b> (1): 13637. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2018NatSR...813637T">2018NatSR...813637T</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1038%2Fs41598-018-31765-z">10.1038/s41598-018-31765-z</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6134142">6134142</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/30206240">30206240</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Sci.+Rep.&rft.atitle=Neurotransmitter+identity+and+electrophysiological+phenotype+are+genetically+coupled+in+midbrain+dopaminergic+neurons&rft.volume=8&rft.issue=1&rft.pages=13637&rft.date=2018&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6134142%23id-name%3DPMC&rft_id=info%3Apmid%2F30206240&rft_id=info%3Adoi%2F10.1038%2Fs41598-018-31765-z&rft_id=info%3Abibcode%2F2018NatSR...813637T&rft.aulast=Tapia&rft.aufirst=M.&rft.au=Baudot%2C+P.&rft.au=Formizano-Treziny%2C+C.&rft.au=Dufour%2C+M.&rft.au=Goaillard%2C+J.M.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6134142&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-18"><span class="mw-cite-backlink"><b><a href="#cite_ref-18">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChristopher_D._ManningPrabhakar_RaghavanHinrich_Schütze2008" class="citation book cs1">Christopher D. Manning; Prabhakar Raghavan; Hinrich Schütze (2008). <i>An Introduction to Information Retrieval</i>. <a href="/wiki/Cambridge_University_Press" title="Cambridge University Press">Cambridge University Press</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-521-86571-5" title="Special:BookSources/978-0-521-86571-5"><bdi>978-0-521-86571-5</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=An+Introduction+to+Information+Retrieval&rft.pub=Cambridge+University+Press&rft.date=2008&rft.isbn=978-0-521-86571-5&rft.au=Christopher+D.+Manning&rft.au=Prabhakar+Raghavan&rft.au=Hinrich+Sch%C3%BCtze&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-19"><span class="mw-cite-backlink"><b><a href="#cite_ref-19">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHaghighatAghagolzadehSeyedarabi2011" class="citation journal cs1">Haghighat, M. B. A.; Aghagolzadeh, A.; Seyedarabi, H. (2011). "A non-reference image fusion metric based on mutual information of image features". <i>Computers & Electrical Engineering</i>. <b>37</b> (5): <span class="nowrap">744–</span>756. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.compeleceng.2011.07.012">10.1016/j.compeleceng.2011.07.012</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:7738541">7738541</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Computers+%26+Electrical+Engineering&rft.atitle=A+non-reference+image+fusion+metric+based+on+mutual+information+of+image+features&rft.volume=37&rft.issue=5&rft.pages=%3Cspan+class%3D%22nowrap%22%3E744-%3C%2Fspan%3E756&rft.date=2011&rft_id=info%3Adoi%2F10.1016%2Fj.compeleceng.2011.07.012&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A7738541%23id-name%3DS2CID&rft.aulast=Haghighat&rft.aufirst=M.+B.+A.&rft.au=Aghagolzadeh%2C+A.&rft.au=Seyedarabi%2C+H.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-20"><span class="mw-cite-backlink"><b><a href="#cite_ref-20">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="http://www.mathworks.com/matlabcentral/fileexchange/45926-feature-mutual-information-fmi-image-fusion-metric">"Feature Mutual Information (FMI) metric for non-reference image fusion - File Exchange - MATLAB Central"</a>. <i>www.mathworks.com</i><span class="reference-accessdate">. Retrieved <span class="nowrap">4 April</span> 2018</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=www.mathworks.com&rft.atitle=Feature+Mutual+Information+%28FMI%29+metric+for+non-reference+image+fusion+-+File+Exchange+-+MATLAB+Central&rft_id=http%3A%2F%2Fwww.mathworks.com%2Fmatlabcentral%2Ffileexchange%2F45926-feature-mutual-information-fmi-image-fusion-metric&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-21"><span class="mw-cite-backlink"><b><a href="#cite_ref-21">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://infotopo.readthedocs.io/en/latest/index.html">"InfoTopo: Topological Information Data Analysis. Deep statistical unsupervised and supervised learning - File Exchange - Github"</a>. <i>github.com/pierrebaudot/infotopopy/</i><span class="reference-accessdate">. Retrieved <span class="nowrap">26 September</span> 2020</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=github.com%2Fpierrebaudot%2Finfotopopy%2F&rft.atitle=InfoTopo%3A+Topological+Information+Data+Analysis.+Deep+statistical+unsupervised+and+supervised+learning+-+File+Exchange+-+Github&rft_id=https%3A%2F%2Finfotopo.readthedocs.io%2Fen%2Flatest%2Findex.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-22"><span class="mw-cite-backlink"><b><a href="#cite_ref-22">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMassey1990" class="citation conference cs1">Massey, James (1990). "Causality, Feedback And Directed Informatio". <i>Proc. 1990 Intl. Symp. on Info. Th. and its Applications, Waikiki, Hawaii, Nov. 27-30, 1990</i>. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.36.5688">10.1.1.36.5688</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Causality%2C+Feedback+And+Directed+Informatio&rft.btitle=Proc.+1990+Intl.+Symp.+on+Info.+Th.+and+its+Applications%2C+Waikiki%2C+Hawaii%2C+Nov.+27-30%2C+1990&rft.date=1990&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.36.5688%23id-name%3DCiteSeerX&rft.aulast=Massey&rft.aufirst=James&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-23"><span class="mw-cite-backlink"><b><a href="#cite_ref-23">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPermuterWeissmanGoldsmith2009" class="citation journal cs1">Permuter, Haim Henry; Weissman, Tsachy; Goldsmith, Andrea J. (February 2009). "Finite State Channels With Time-Invariant Deterministic Feedback". <i>IEEE Transactions on Information Theory</i>. <b>55</b> (2): <span class="nowrap">644–</span>662. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/cs/0608070">cs/0608070</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTIT.2008.2009849">10.1109/TIT.2008.2009849</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:13178">13178</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Information+Theory&rft.atitle=Finite+State+Channels+With+Time-Invariant+Deterministic+Feedback&rft.volume=55&rft.issue=2&rft.pages=%3Cspan+class%3D%22nowrap%22%3E644-%3C%2Fspan%3E662&rft.date=2009-02&rft_id=info%3Aarxiv%2Fcs%2F0608070&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A13178%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FTIT.2008.2009849&rft.aulast=Permuter&rft.aufirst=Haim+Henry&rft.au=Weissman%2C+Tsachy&rft.au=Goldsmith%2C+Andrea+J.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-FOOTNOTECoombsDawesTversky1970-24"><span class="mw-cite-backlink"><b><a href="#cite_ref-FOOTNOTECoombsDawesTversky1970_24-0">^</a></b></span> <span class="reference-text"><a href="#CITEREFCoombsDawesTversky1970">Coombs, Dawes & Tversky 1970</a>.</span> </li> <li id="cite_note-pressflannery-25"><span class="mw-cite-backlink">^ <a href="#cite_ref-pressflannery_25-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-pressflannery_25-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPressTeukolskyVetterlingFlannery2007" class="citation book cs1">Press, WH; Teukolsky, SA; Vetterling, WT; Flannery, BP (2007). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20110811154417/http://apps.nrbook.com/empanel/index.html#pg=758">"Section 14.7.3. Conditional Entropy and Mutual Information"</a>. <i>Numerical Recipes: The Art of Scientific Computing</i> (3rd ed.). New York: Cambridge University Press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-521-88068-8" title="Special:BookSources/978-0-521-88068-8"><bdi>978-0-521-88068-8</bdi></a>. Archived from <a rel="nofollow" class="external text" href="http://apps.nrbook.com/empanel/index.html#pg=758">the original</a> on 2011-08-11<span class="reference-accessdate">. Retrieved <span class="nowrap">2011-08-13</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Section+14.7.3.+Conditional+Entropy+and+Mutual+Information&rft.btitle=Numerical+Recipes%3A+The+Art+of+Scientific+Computing&rft.place=New+York&rft.edition=3rd&rft.pub=Cambridge+University+Press&rft.date=2007&rft.isbn=978-0-521-88068-8&rft.aulast=Press&rft.aufirst=WH&rft.au=Teukolsky%2C+SA&rft.au=Vetterling%2C+WT&rft.au=Flannery%2C+BP&rft_id=http%3A%2F%2Fapps.nrbook.com%2Fempanel%2Findex.html%23pg%3D758&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-JimWhite-26"><span class="mw-cite-backlink"><b><a href="#cite_ref-JimWhite_26-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWhiteSteingoldFournelle" class="citation conference cs1 cs1-prop-unfit">White, Jim; Steingold, Sam; Fournelle, Connie. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20160705074827/http://www.interfacesymposia.org/I04/I2004Proceedings/WhiteJim/WhiteJim.paper.pdf"><i>Performance Metrics for Group-Detection Algorithms</i></a> <span class="cs1-format">(PDF)</span>. Interface 2004. Archived from the original on 2016-07-05<span class="reference-accessdate">. Retrieved <span class="nowrap">2014-02-19</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.btitle=Performance+Metrics+for+Group-Detection+Algorithms&rft.aulast=White&rft.aufirst=Jim&rft.au=Steingold%2C+Sam&rft.au=Fournelle%2C+Connie&rft_id=http%3A%2F%2Fwww.interfacesymposia.org%2FI04%2FI2004Proceedings%2FWhiteJim%2FWhiteJim.paper.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-DRWijaya-27"><span class="mw-cite-backlink"><b><a href="#cite_ref-DRWijaya_27-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWijayaSarnoZulaika2017" class="citation journal cs1">Wijaya, Dedy Rahman; Sarno, Riyanarto; Zulaika, Enny (2017). "Information Quality Ratio as a novel metric for mother wavelet selection". <i>Chemometrics and Intelligent Laboratory Systems</i>. <b>160</b>: <span class="nowrap">59–</span>71. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.chemolab.2016.11.012">10.1016/j.chemolab.2016.11.012</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Chemometrics+and+Intelligent+Laboratory+Systems&rft.atitle=Information+Quality+Ratio+as+a+novel+metric+for+mother+wavelet+selection&rft.volume=160&rft.pages=%3Cspan+class%3D%22nowrap%22%3E59-%3C%2Fspan%3E71&rft.date=2017&rft_id=info%3Adoi%2F10.1016%2Fj.chemolab.2016.11.012&rft.aulast=Wijaya&rft.aufirst=Dedy+Rahman&rft.au=Sarno%2C+Riyanarto&rft.au=Zulaika%2C+Enny&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-strehl-jmlr02-28"><span class="mw-cite-backlink"><b><a href="#cite_ref-strehl-jmlr02_28-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFStrehlGhosh2003" class="citation journal cs1">Strehl, Alexander; Ghosh, Joydeep (2003). <a rel="nofollow" class="external text" href="http://www.jmlr.org/papers/volume3/strehl02a/strehl02a.pdf">"Cluster Ensembles – A Knowledge Reuse Framework for Combining Multiple Partitions"</a> <span class="cs1-format">(PDF)</span>. <i>The Journal of Machine Learning Research</i>. <b>3</b>: <span class="nowrap">583–</span>617. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1162%2F153244303321897735">10.1162/153244303321897735</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Journal+of+Machine+Learning+Research&rft.atitle=Cluster+Ensembles+%E2%80%93+A+Knowledge+Reuse+Framework+for+Combining+Multiple+Partitions&rft.volume=3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E583-%3C%2Fspan%3E617&rft.date=2003&rft_id=info%3Adoi%2F10.1162%2F153244303321897735&rft.aulast=Strehl&rft.aufirst=Alexander&rft.au=Ghosh%2C+Joydeep&rft_id=http%3A%2F%2Fwww.jmlr.org%2Fpapers%2Fvolume3%2Fstrehl02a%2Fstrehl02a.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-weighted-kl-29"><span class="mw-cite-backlink"><b><a href="#cite_ref-weighted-kl_29-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKvålseth1991" class="citation journal cs1">Kvålseth, T. O. (1991). "The relative useful information measure: some comments". <i>Information Sciences</i>. <b>56</b> (1): <span class="nowrap">35–</span>38. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2F0020-0255%2891%2990022-m">10.1016/0020-0255(91)90022-m</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Information+Sciences&rft.atitle=The+relative+useful+information+measure%3A+some+comments&rft.volume=56&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E35-%3C%2Fspan%3E38&rft.date=1991&rft_id=info%3Adoi%2F10.1016%2F0020-0255%2891%2990022-m&rft.aulast=Kv%C3%A5lseth&rft.aufirst=T.+O.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-30"><span class="mw-cite-backlink"><b><a href="#cite_ref-30">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPocock2012" class="citation thesis cs1">Pocock, A. (2012). <a rel="nofollow" class="external text" href="http://www.cs.man.ac.uk/~gbrown/publications/pocockPhDthesis.pdf"><i>Feature Selection Via Joint Likelihood</i></a> <span class="cs1-format">(PDF)</span> (Thesis).</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Adissertation&rft.title=Feature+Selection+Via+Joint+Likelihood&rft.date=2012&rft.aulast=Pocock&rft.aufirst=A.&rft_id=http%3A%2F%2Fwww.cs.man.ac.uk%2F~gbrown%2Fpublications%2FpocockPhDthesis.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-magerman-31"><span class="mw-cite-backlink">^ <a href="#cite_ref-magerman_31-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-magerman_31-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><a rel="nofollow" class="external text" href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.78.4178&rep=rep1&type=pdf">Parsing a Natural Language Using Mutual Information Statistics</a> by David M. Magerman and Mitchell P. Marcus</span> </li> <li id="cite_note-everett56-32"><span class="mw-cite-backlink"><b><a href="#cite_ref-everett56_32-0">^</a></b></span> <span class="reference-text"><a href="/wiki/Hugh_Everett" class="mw-redirect" title="Hugh Everett">Hugh Everett</a> <a rel="nofollow" class="external text" href="https://www.pbs.org/wgbh/nova/manyworlds/pdf/dissertation.pdf">Theory of the Universal Wavefunction</a>, Thesis, Princeton University, (1956, 1973), pp 1–140 (page 30)</span> </li> <li id="cite_note-everett57-33"><span class="mw-cite-backlink"><b><a href="#cite_ref-everett57_33-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFEverett1957" class="citation journal cs1"><a href="/wiki/Hugh_Everett" class="mw-redirect" title="Hugh Everett">Everett, Hugh</a> (1957). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20111027191052/http://www.univer.omsk.su/omsk/Sci/Everett/paper1957.html">"Relative State Formulation of Quantum Mechanics"</a>. <i>Reviews of Modern Physics</i>. <b>29</b> (3): <span class="nowrap">454–</span>462. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1957RvMP...29..454E">1957RvMP...29..454E</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2Frevmodphys.29.454">10.1103/revmodphys.29.454</a>. Archived from <a rel="nofollow" class="external text" href="http://www.univer.omsk.su/omsk/Sci/Everett/paper1957.html">the original</a> on 2011-10-27<span class="reference-accessdate">. Retrieved <span class="nowrap">2012-07-16</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Reviews+of+Modern+Physics&rft.atitle=Relative+State+Formulation+of+Quantum+Mechanics&rft.volume=29&rft.issue=3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E454-%3C%2Fspan%3E462&rft.date=1957&rft_id=info%3Adoi%2F10.1103%2Frevmodphys.29.454&rft_id=info%3Abibcode%2F1957RvMP...29..454E&rft.aulast=Everett&rft.aufirst=Hugh&rft_id=http%3A%2F%2Fwww.univer.omsk.su%2Fomsk%2FSci%2FEverett%2Fpaper1957.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-34"><span class="mw-cite-backlink"><b><a href="#cite_ref-34">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFNicolettiBusiello2021" class="citation journal cs1">Nicoletti, Giorgio; Busiello, Daniel Maria (2021-11-22). <a rel="nofollow" class="external text" href="https://link.aps.org/doi/10.1103/PhysRevLett.127.228301">"Mutual Information Disentangles Interactions from Changing Environments"</a>. <i>Physical Review Letters</i>. <b>127</b> (22): 228301. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2107.08985">2107.08985</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2021PhRvL.127v8301N">2021PhRvL.127v8301N</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevLett.127.228301">10.1103/PhysRevLett.127.228301</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/34889638">34889638</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:236087228">236087228</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+Letters&rft.atitle=Mutual+Information+Disentangles+Interactions+from+Changing+Environments&rft.volume=127&rft.issue=22&rft.pages=228301&rft.date=2021-11-22&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A236087228%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2021PhRvL.127v8301N&rft_id=info%3Aarxiv%2F2107.08985&rft_id=info%3Apmid%2F34889638&rft_id=info%3Adoi%2F10.1103%2FPhysRevLett.127.228301&rft.aulast=Nicoletti&rft.aufirst=Giorgio&rft.au=Busiello%2C+Daniel+Maria&rft_id=https%3A%2F%2Flink.aps.org%2Fdoi%2F10.1103%2FPhysRevLett.127.228301&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-35"><span class="mw-cite-backlink"><b><a href="#cite_ref-35">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFNicolettiBusiello2022" class="citation journal cs1">Nicoletti, Giorgio; Busiello, Daniel Maria (2022-07-29). <a rel="nofollow" class="external text" href="https://link.aps.org/doi/10.1103/PhysRevE.106.014153">"Mutual information in changing environments: Nonlinear interactions, out-of-equilibrium systems, and continuously varying diffusivities"</a>. <i>Physical Review E</i>. <b>106</b> (1): 014153. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2204.01644">2204.01644</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2022PhRvE.106a4153N">2022PhRvE.106a4153N</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevE.106.014153">10.1103/PhysRevE.106.014153</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/35974654">35974654</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+E&rft.atitle=Mutual+information+in+changing+environments%3A+Nonlinear+interactions%2C+out-of-equilibrium+systems%2C+and+continuously+varying+diffusivities&rft.volume=106&rft.issue=1&rft.pages=014153&rft.date=2022-07-29&rft_id=info%3Aarxiv%2F2204.01644&rft_id=info%3Apmid%2F35974654&rft_id=info%3Adoi%2F10.1103%2FPhysRevE.106.014153&rft_id=info%3Abibcode%2F2022PhRvE.106a4153N&rft.aulast=Nicoletti&rft.aufirst=Giorgio&rft.au=Busiello%2C+Daniel+Maria&rft_id=https%3A%2F%2Flink.aps.org%2Fdoi%2F10.1103%2FPhysRevE.106.014153&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-36"><span class="mw-cite-backlink"><b><a href="#cite_ref-36">^</a></b></span> <span class="reference-text"><a rel="nofollow" class="external text" href="https://code.google.com/p/globalmit">GlobalMIT</a> at <a href="/wiki/Google_Code" class="mw-redirect" title="Google Code">Google Code</a></span> </li> <li id="cite_note-37"><span class="mw-cite-backlink"><b><a href="#cite_ref-37">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLee2021" class="citation journal cs1">Lee, Se Yoon (2021). "Gibbs sampler and coordinate ascent variational inference: A set-theoretical review". <i>Communications in Statistics - Theory and Methods</i>. <b>51</b> (6): <span class="nowrap">1549–</span>1568. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2008.01006">2008.01006</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F03610926.2021.1921214">10.1080/03610926.2021.1921214</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:220935477">220935477</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Communications+in+Statistics+-+Theory+and+Methods&rft.atitle=Gibbs+sampler+and+coordinate+ascent+variational+inference%3A+A+set-theoretical+review&rft.volume=51&rft.issue=6&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1549-%3C%2Fspan%3E1568&rft.date=2021&rft_id=info%3Aarxiv%2F2008.01006&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A220935477%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1080%2F03610926.2021.1921214&rft.aulast=Lee&rft.aufirst=Se+Yoon&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-38"><span class="mw-cite-backlink"><b><a href="#cite_ref-38">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKeysKholikovPevtsov2015" class="citation journal cs1">Keys, Dustin; Kholikov, Shukur; Pevtsov, Alexei A. (February 2015). "Application of Mutual Information Methods in Time Distance Helioseismology". <i>Solar Physics</i>. <b>290</b> (3): <span class="nowrap">659–</span>671. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1501.05597">1501.05597</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2015SoPh..290..659K">2015SoPh..290..659K</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs11207-015-0650-y">10.1007/s11207-015-0650-y</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:118472242">118472242</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Solar+Physics&rft.atitle=Application+of+Mutual+Information+Methods+in+Time+Distance+Helioseismology&rft.volume=290&rft.issue=3&rft.pages=%3Cspan+class%3D%22nowrap%22%3E659-%3C%2Fspan%3E671&rft.date=2015-02&rft_id=info%3Aarxiv%2F1501.05597&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A118472242%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1007%2Fs11207-015-0650-y&rft_id=info%3Abibcode%2F2015SoPh..290..659K&rft.aulast=Keys&rft.aufirst=Dustin&rft.au=Kholikov%2C+Shukur&rft.au=Pevtsov%2C+Alexei+A.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> <li id="cite_note-iic-39"><span class="mw-cite-backlink"><b><a href="#cite_ref-iic_39-0">^</a></b></span> <span class="reference-text"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1807.06653">Invariant Information Clustering for Unsupervised Image Classification and Segmentation</a> by Xu Ji, Joao Henriques and Andrea Vedaldi</span> </li> <li id="cite_note-40"><span class="mw-cite-backlink"><b><a href="#cite_ref-40">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFNicolettiBusiello2024" class="citation journal cs1">Nicoletti, Giorgio; Busiello, Daniel Maria (2024-04-08). <a rel="nofollow" class="external text" href="https://link.aps.org/doi/10.1103/PhysRevX.14.021007">"Information Propagation in Multilayer Systems with Higher-Order Interactions across Timescales"</a>. <i>Physical Review X</i>. <b>14</b> (2): 021007. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2312.06246">2312.06246</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2024PhRvX..14b1007N">2024PhRvX..14b1007N</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2FPhysRevX.14.021007">10.1103/PhysRevX.14.021007</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+X&rft.atitle=Information+Propagation+in+Multilayer+Systems+with+Higher-Order+Interactions+across+Timescales&rft.volume=14&rft.issue=2&rft.pages=021007&rft.date=2024-04-08&rft_id=info%3Aarxiv%2F2312.06246&rft_id=info%3Adoi%2F10.1103%2FPhysRevX.14.021007&rft_id=info%3Abibcode%2F2024PhRvX..14b1007N&rft.aulast=Nicoletti&rft.aufirst=Giorgio&rft.au=Busiello%2C+Daniel+Maria&rft_id=https%3A%2F%2Flink.aps.org%2Fdoi%2F10.1103%2FPhysRevX.14.021007&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></span> </li> </ol></div></div> <div class="mw-heading mw-heading2"><h2 id="References">References</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Mutual_information&action=edit&section=29" title="Edit section: References"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBaudotTapiaBennequinGoaillard2019" class="citation journal cs1">Baudot, P.; Tapia, M.; Bennequin, D.; Goaillard, J.M. (2019). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7515398">"Topological Information Data Analysis"</a>. <i>Entropy</i>. <b>21</b> (9). 869. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1907.04242">1907.04242</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2019Entrp..21..869B">2019Entrp..21..869B</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.3390%2Fe21090869">10.3390/e21090869</a></span>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7515398">7515398</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:195848308">195848308</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Entropy&rft.atitle=Topological+Information+Data+Analysis&rft.volume=21&rft.issue=9&rft.pages=869&rft.date=2019&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC7515398%23id-name%3DPMC&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A195848308%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2019Entrp..21..869B&rft_id=info%3Aarxiv%2F1907.04242&rft_id=info%3Adoi%2F10.3390%2Fe21090869&rft.aulast=Baudot&rft.aufirst=P.&rft.au=Tapia%2C+M.&rft.au=Bennequin%2C+D.&rft.au=Goaillard%2C+J.M.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC7515398&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCilibrasiVitányi2005" class="citation journal cs1">Cilibrasi, R.; Vitányi, Paul (2005). <a rel="nofollow" class="external text" href="http://www.cwi.nl/~paulv/papers/cluster.pdf">"Clustering by compression"</a> <span class="cs1-format">(PDF)</span>. <i>IEEE Transactions on Information Theory</i>. <b>51</b> (4): <span class="nowrap">1523–</span>1545. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/cs/0312044">cs/0312044</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FTIT.2005.844059">10.1109/TIT.2005.844059</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:911">911</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Information+Theory&rft.atitle=Clustering+by+compression&rft.volume=51&rft.issue=4&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1523-%3C%2Fspan%3E1545&rft.date=2005&rft_id=info%3Aarxiv%2Fcs%2F0312044&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A911%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2FTIT.2005.844059&rft.aulast=Cilibrasi&rft.aufirst=R.&rft.au=Vit%C3%A1nyi%2C+Paul&rft_id=http%3A%2F%2Fwww.cwi.nl%2F~paulv%2Fpapers%2Fcluster.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCronbach1954" class="citation book cs1">Cronbach, L. J. (1954). "On the non-rational application of information measures in psychology". In <a href="/wiki/Henry_Quastler" title="Henry Quastler">Quastler, Henry</a> (ed.). <i>Information Theory in Psychology: Problems and Methods</i>. Glencoe, Illinois: Free Press. pp. <span class="nowrap">14–</span>30.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=On+the+non-rational+application+of+information+measures+in+psychology&rft.btitle=Information+Theory+in+Psychology%3A+Problems+and+Methods&rft.place=Glencoe%2C+Illinois&rft.pages=%3Cspan+class%3D%22nowrap%22%3E14-%3C%2Fspan%3E30&rft.pub=Free+Press&rft.date=1954&rft.aulast=Cronbach&rft.aufirst=L.+J.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCoombsDawesTversky1970" class="citation book cs1">Coombs, C. H.; Dawes, R. M.; Tversky, A. (1970). <i>Mathematical Psychology: An Elementary Introduction</i>. Englewood Cliffs, New Jersey: Prentice-Hall.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Mathematical+Psychology%3A+An+Elementary+Introduction&rft.place=Englewood+Cliffs%2C+New+Jersey&rft.pub=Prentice-Hall&rft.date=1970&rft.aulast=Coombs&rft.aufirst=C.+H.&rft.au=Dawes%2C+R.+M.&rft.au=Tversky%2C+A.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChurchHanks1989" class="citation journal cs1">Church, Kenneth Ward; Hanks, Patrick (1989). <a rel="nofollow" class="external text" href="https://doi.org/10.3115%2F981623.981633">"Word association norms, mutual information, and lexicography"</a>. <i>Proceedings of the 27th Annual Meeting of the Association for Computational Linguistics</i>. <b>16</b> (1): <span class="nowrap">76–</span>83. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.3115%2F981623.981633">10.3115/981623.981633</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Proceedings+of+the+27th+Annual+Meeting+of+the+Association+for+Computational+Linguistics&rft.atitle=Word+association+norms%2C+mutual+information%2C+and+lexicography&rft.volume=16&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E76-%3C%2Fspan%3E83&rft.date=1989&rft_id=info%3Adoi%2F10.3115%2F981623.981633&rft.aulast=Church&rft.aufirst=Kenneth+Ward&rft.au=Hanks%2C+Patrick&rft_id=https%3A%2F%2Fdoi.org%2F10.3115%252F981623.981633&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGel'fandYaglom1957" class="citation journal cs1">Gel'fand, I.M.; Yaglom, A.M. (1957). "Calculation of amount of information about a random function contained in another such function". <i>American Mathematical Society Translations</i>. Series 2. <b>12</b>: <span class="nowrap">199–</span>246. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1090%2Ftrans2%2F012%2F09">10.1090/trans2/012/09</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9780821817124" title="Special:BookSources/9780821817124"><bdi>9780821817124</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=American+Mathematical+Society+Translations&rft.atitle=Calculation+of+amount+of+information+about+a+random+function+contained+in+another+such+function&rft.volume=12&rft.pages=%3Cspan+class%3D%22nowrap%22%3E199-%3C%2Fspan%3E246&rft.date=1957&rft_id=info%3Adoi%2F10.1090%2Ftrans2%2F012%2F09&rft.isbn=9780821817124&rft.aulast=Gel%27fand&rft.aufirst=I.M.&rft.au=Yaglom%2C+A.M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span> English translation of original in <i>Uspekhi Matematicheskikh Nauk</i> <b>12</b> (1): 3-52.</li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGuiasu1977" class="citation book cs1">Guiasu, Silviu (1977). <i>Information Theory with Applications</i>. McGraw-Hill, New York. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-07-025109-0" title="Special:BookSources/978-0-07-025109-0"><bdi>978-0-07-025109-0</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Information+Theory+with+Applications&rft.pub=McGraw-Hill%2C+New+York&rft.date=1977&rft.isbn=978-0-07-025109-0&rft.aulast=Guiasu&rft.aufirst=Silviu&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLiVitányi1997" class="citation book cs1">Li, Ming; Vitányi, Paul (February 1997). <i>An introduction to Kolmogorov complexity and its applications</i>. New York: <a href="/wiki/Springer-Verlag" class="mw-redirect" title="Springer-Verlag">Springer-Verlag</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-387-94868-3" title="Special:BookSources/978-0-387-94868-3"><bdi>978-0-387-94868-3</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=An+introduction+to+Kolmogorov+complexity+and+its+applications&rft.place=New+York&rft.pub=Springer-Verlag&rft.date=1997-02&rft.isbn=978-0-387-94868-3&rft.aulast=Li&rft.aufirst=Ming&rft.au=Vit%C3%A1nyi%2C+Paul&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLockhead1970" class="citation journal cs1">Lockhead, G. R. (1970). "Identification and the form of multidimensional discrimination space". <i>Journal of Experimental Psychology</i>. <b>85</b> (1): <span class="nowrap">1–</span>10. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1037%2Fh0029508">10.1037/h0029508</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/5458322">5458322</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Experimental+Psychology&rft.atitle=Identification+and+the+form+of+multidimensional+discrimination+space&rft.volume=85&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1-%3C%2Fspan%3E10&rft.date=1970&rft_id=info%3Adoi%2F10.1037%2Fh0029508&rft_id=info%3Apmid%2F5458322&rft.aulast=Lockhead&rft.aufirst=G.+R.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li>David J. C. MacKay. <i><a rel="nofollow" class="external text" href="http://www.inference.phy.cam.ac.uk/mackay/itila/book.html">Information Theory, Inference, and Learning Algorithms</a></i> Cambridge: Cambridge University Press, 2003. <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/0-521-64298-1" title="Special:BookSources/0-521-64298-1">0-521-64298-1</a> (available free online)</li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHaghighatAghagolzadehSeyedarabi2011" class="citation journal cs1">Haghighat, M. B. A.; Aghagolzadeh, A.; Seyedarabi, H. (2011). "A non-reference image fusion metric based on mutual information of image features". <i>Computers & Electrical Engineering</i>. <b>37</b> (5): <span class="nowrap">744–</span>756. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.compeleceng.2011.07.012">10.1016/j.compeleceng.2011.07.012</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:7738541">7738541</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Computers+%26+Electrical+Engineering&rft.atitle=A+non-reference+image+fusion+metric+based+on+mutual+information+of+image+features&rft.volume=37&rft.issue=5&rft.pages=%3Cspan+class%3D%22nowrap%22%3E744-%3C%2Fspan%3E756&rft.date=2011&rft_id=info%3Adoi%2F10.1016%2Fj.compeleceng.2011.07.012&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A7738541%23id-name%3DS2CID&rft.aulast=Haghighat&rft.aufirst=M.+B.+A.&rft.au=Aghagolzadeh%2C+A.&rft.au=Seyedarabi%2C+H.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><a href="/wiki/Athanasios_Papoulis" title="Athanasios Papoulis">Athanasios Papoulis</a>. <i>Probability, Random Variables, and Stochastic Processes</i>, second edition. New York: McGraw-Hill, 1984. <i>(See Chapter 15.)</i></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWittenFrank2005" class="citation book cs1">Witten, Ian H. & Frank, Eibe (2005). <a rel="nofollow" class="external text" href="http://www.cs.waikato.ac.nz/~ml/weka/book.html"><i>Data Mining: Practical Machine Learning Tools and Techniques</i></a>. Morgan Kaufmann, Amsterdam. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-12-374856-0" title="Special:BookSources/978-0-12-374856-0"><bdi>978-0-12-374856-0</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Data+Mining%3A+Practical+Machine+Learning+Tools+and+Techniques&rft.pub=Morgan+Kaufmann%2C+Amsterdam&rft.date=2005&rft.isbn=978-0-12-374856-0&rft.aulast=Witten&rft.aufirst=Ian+H.&rft.au=Frank%2C+Eibe&rft_id=http%3A%2F%2Fwww.cs.waikato.ac.nz%2F~ml%2Fweka%2Fbook.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPeng,_H.C.Long,_F.Ding,_C.2005" class="citation journal cs1">Peng, H.C.; Long, F. & Ding, C. (2005). <a rel="nofollow" class="external text" href="http://research.janelia.org/peng/proj/mRMR/index.htm">"Feature selection based on mutual information: criteria of max-dependency, max-relevance, and min-redundancy"</a>. <i>IEEE Transactions on Pattern Analysis and Machine Intelligence</i>. <b>27</b> (8): <span class="nowrap">1226–</span>1238. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.63.5765">10.1.1.63.5765</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1109%2Ftpami.2005.159">10.1109/tpami.2005.159</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/16119262">16119262</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:206764015">206764015</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Transactions+on+Pattern+Analysis+and+Machine+Intelligence&rft.atitle=Feature+selection+based+on+mutual+information%3A+criteria+of+max-dependency%2C+max-relevance%2C+and+min-redundancy&rft.volume=27&rft.issue=8&rft.pages=%3Cspan+class%3D%22nowrap%22%3E1226-%3C%2Fspan%3E1238&rft.date=2005&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.63.5765%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A206764015%23id-name%3DS2CID&rft_id=info%3Apmid%2F16119262&rft_id=info%3Adoi%2F10.1109%2Ftpami.2005.159&rft.au=Peng%2C+H.C.&rft.au=Long%2C+F.&rft.au=Ding%2C+C.&rft_id=http%3A%2F%2Fresearch.janelia.org%2Fpeng%2Fproj%2FmRMR%2Findex.htm&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFAndre_S._RibeiroStuart_A._KauffmanJason_Lloyd-PriceBjorn_Samuelsson2008" class="citation journal cs1">Andre S. Ribeiro; Stuart A. Kauffman; Jason Lloyd-Price; Bjorn Samuelsson & Joshua Socolar (2008). "Mutual Information in Random Boolean models of regulatory networks". <i>Physical Review E</i>. <b>77</b> (1): 011901. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/0707.3642">0707.3642</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2008PhRvE..77a1901R">2008PhRvE..77a1901R</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1103%2Fphysreve.77.011901">10.1103/physreve.77.011901</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/18351870">18351870</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15232112">15232112</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Review+E&rft.atitle=Mutual+Information+in+Random+Boolean+models+of+regulatory+networks&rft.volume=77&rft.issue=1&rft.pages=011901&rft.date=2008&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15232112%23id-name%3DS2CID&rft_id=info%3Abibcode%2F2008PhRvE..77a1901R&rft_id=info%3Aarxiv%2F0707.3642&rft_id=info%3Apmid%2F18351870&rft_id=info%3Adoi%2F10.1103%2Fphysreve.77.011901&rft.au=Andre+S.+Ribeiro&rft.au=Stuart+A.+Kauffman&rft.au=Jason+Lloyd-Price&rft.au=Bjorn+Samuelsson&rft.au=Joshua+Socolar&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWellsViolaAtsumiNakajima1996" class="citation journal cs1">Wells, W.M. III; Viola, P.; Atsumi, H.; Nakajima, S.; Kikinis, R. (1996). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20080906201633/http://www.ai.mit.edu/people/sw/papers/mia.pdf">"Multi-modal volume registration by maximization of mutual information"</a> <span class="cs1-format">(PDF)</span>. <i>Medical Image Analysis</i>. <b>1</b> (1): <span class="nowrap">35–</span>51. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS1361-8415%2801%2980004-9">10.1016/S1361-8415(01)80004-9</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/9873920">9873920</a>. Archived from <a rel="nofollow" class="external text" href="http://www.ai.mit.edu/people/sw/papers/mia.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 2008-09-06<span class="reference-accessdate">. Retrieved <span class="nowrap">2010-08-05</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Medical+Image+Analysis&rft.atitle=Multi-modal+volume+registration+by+maximization+of+mutual+information&rft.volume=1&rft.issue=1&rft.pages=%3Cspan+class%3D%22nowrap%22%3E35-%3C%2Fspan%3E51&rft.date=1996&rft_id=info%3Adoi%2F10.1016%2FS1361-8415%2801%2980004-9&rft_id=info%3Apmid%2F9873920&rft.aulast=Wells&rft.aufirst=W.M.+III&rft.au=Viola%2C+P.&rft.au=Atsumi%2C+H.&rft.au=Nakajima%2C+S.&rft.au=Kikinis%2C+R.&rft_id=http%3A%2F%2Fwww.ai.mit.edu%2Fpeople%2Fsw%2Fpapers%2Fmia.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPandeySarkar2017" class="citation journal cs1">Pandey, Biswajit; Sarkar, Suman (2017). <a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fmnrasl%2Fslw250">"How much a galaxy knows about its large-scale environment?: An information theoretic perspective"</a>. <i>Monthly Notices of the Royal Astronomical Society Letters</i>. <b>467</b> (1): L6. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1611.00283">1611.00283</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2017MNRAS.467L...6P">2017MNRAS.467L...6P</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fmnrasl%2Fslw250">10.1093/mnrasl/slw250</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:119095496">119095496</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Monthly+Notices+of+the+Royal+Astronomical+Society+Letters&rft.atitle=How+much+a+galaxy+knows+about+its+large-scale+environment%3F%3A+An+information+theoretic+perspective&rft.volume=467&rft.issue=1&rft.pages=L6&rft.date=2017&rft_id=info%3Aarxiv%2F1611.00283&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A119095496%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1093%2Fmnrasl%2Fslw250&rft_id=info%3Abibcode%2F2017MNRAS.467L...6P&rft.aulast=Pandey&rft.aufirst=Biswajit&rft.au=Sarkar%2C+Suman&rft_id=https%3A%2F%2Fdoi.org%2F10.1093%252Fmnrasl%252Fslw250&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li></ul> <ul><li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJanssenGuanRobeva2023" class="citation journal cs1">Janssen, Joseph; Guan, Vincent; Robeva, Elina (2023). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v206/janssen23a.html">"Ultra-marginal Feature Importance: Learning from Data with Causal Guarantees"</a>. <i>International Conference on Artificial Intelligence and Statistics</i>: <span class="nowrap">10782–</span>10814. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2204.09938">2204.09938</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=International+Conference+on+Artificial+Intelligence+and+Statistics&rft.atitle=Ultra-marginal+Feature+Importance%3A+Learning+from+Data+with+Causal+Guarantees&rft.pages=%3Cspan+class%3D%22nowrap%22%3E10782-%3C%2Fspan%3E10814&rft.date=2023&rft_id=info%3Aarxiv%2F2204.09938&rft.aulast=Janssen&rft.aufirst=Joseph&rft.au=Guan%2C+Vincent&rft.au=Robeva%2C+Elina&rft_id=https%3A%2F%2Fproceedings.mlr.press%2Fv206%2Fjanssen23a.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AMutual+information" class="Z3988"></span></li></ul> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox authority-control" aria-label="Navbox390" style="padding:3px"><table class="nowraplinks hlist navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Help:Authority_control" title="Help:Authority control">Authority control databases</a>: National <span class="mw-valign-text-top noprint" typeof="mw:File/Frameless"><a href="https://www.wikidata.org/wiki/Q252973#identifiers" title="Edit this at Wikidata"><img alt="Edit this at Wikidata" src="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/10px-OOjs_UI_icon_edit-ltr-progressive.svg.png" decoding="async" width="10" height="10" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/15px-OOjs_UI_icon_edit-ltr-progressive.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/20px-OOjs_UI_icon_edit-ltr-progressive.svg.png 2x" data-file-width="20" data-file-height="20" /></a></span></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"><ul><li><span class="uid"><a rel="nofollow" class="external text" href="https://d-nb.info/gnd/4779212-7">Germany</a></span></li></ul></div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?useformat=desktop&type=1x1&usesul3=0" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Mutual_information&oldid=1270339045">https://en.wikipedia.org/w/index.php?title=Mutual_information&oldid=1270339045</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Information_theory" title="Category:Information theory">Information theory</a></li><li><a href="/wiki/Category:Entropy_and_information" title="Category:Entropy and information">Entropy and information</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:CS1:_unfit_URL" title="Category:CS1: unfit URL">CS1: unfit URL</a></li><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_matches_Wikidata" title="Category:Short description matches Wikidata">Short description matches Wikidata</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 19 January 2025, at 02:09<span class="anonymous-show"> (UTC)</span>.</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Mutual_information&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" lang="en" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><picture><source media="(min-width: 500px)" srcset="/w/resources/assets/poweredby_mediawiki.svg" width="88" height="31"><img src="/w/resources/assets/mediawiki_compact.svg" alt="Powered by MediaWiki" width="25" height="25" loading="lazy"></picture></a></li> </ul> </footer> </div> </div> </div> <div class="vector-header-container vector-sticky-header-container"> <div id="vector-sticky-header" class="vector-sticky-header"> <div class="vector-sticky-header-start"> <div class="vector-sticky-header-icon-start vector-button-flush-left vector-button-flush-right" aria-hidden="true"> <button class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-sticky-header-search-toggle" tabindex="-1" data-event-name="ui.vector-sticky-search-form.icon"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </button> </div> <div role="search" class="vector-search-box-vue vector-search-box-show-thumbnail vector-search-box"> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail"> <form action="/w/index.php" id="vector-sticky-search-form" class="cdx-search-input cdx-search-input--has-end-button"> <div class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia"> <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <div class="vector-sticky-header-context-bar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-sticky-header-toc" class="vector-dropdown mw-portlet mw-portlet-sticky-header-toc vector-sticky-header-toc vector-button-flush-left" > <input type="checkbox" id="vector-sticky-header-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-sticky-header-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-sticky-header-toc-label" for="vector-sticky-header-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-sticky-header-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div class="vector-sticky-header-context-bar-primary" aria-hidden="true" ><span class="mw-page-title-main">Mutual information</span></div> </div> </div> <div class="vector-sticky-header-end" aria-hidden="true"> <div class="vector-sticky-header-icons"> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-talk-sticky-header" tabindex="-1" data-event-name="talk-sticky-header"><span class="vector-icon mw-ui-icon-speechBubbles mw-ui-icon-wikimedia-speechBubbles"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-subject-sticky-header" tabindex="-1" data-event-name="subject-sticky-header"><span class="vector-icon mw-ui-icon-article mw-ui-icon-wikimedia-article"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-history-sticky-header" tabindex="-1" data-event-name="history-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-history mw-ui-icon-wikimedia-wikimedia-history"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only mw-watchlink" id="ca-watchstar-sticky-header" tabindex="-1" data-event-name="watch-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-star mw-ui-icon-wikimedia-wikimedia-star"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-edit-sticky-header" tabindex="-1" data-event-name="wikitext-edit-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-wikiText mw-ui-icon-wikimedia-wikimedia-wikiText"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-ve-edit-sticky-header" tabindex="-1" data-event-name="ve-edit-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-edit mw-ui-icon-wikimedia-wikimedia-edit"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-viewsource-sticky-header" tabindex="-1" data-event-name="ve-edit-protected-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-editLock mw-ui-icon-wikimedia-wikimedia-editLock"></span> <span></span> </a> </div> <div class="vector-sticky-header-buttons"> <button class="cdx-button cdx-button--weight-quiet mw-interlanguage-selector" id="p-lang-btn-sticky-header" tabindex="-1" data-event-name="ui.dropdown-p-lang-btn-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>20 languages</span> </button> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive" id="ca-addsection-sticky-header" tabindex="-1" data-event-name="addsection-sticky-header"><span class="vector-icon mw-ui-icon-speechBubbleAdd-progressive mw-ui-icon-wikimedia-speechBubbleAdd-progressive"></span> <span>Add topic</span> </a> </div> <div class="vector-sticky-header-icon-end"> <div class="vector-user-links"> </div> </div> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-6b9576b97f-x28md","wgBackendResponseTime":177,"wgPageParseReport":{"limitreport":{"cputime":"0.938","walltime":"1.141","ppvisitednodes":{"value":5584,"limit":1000000},"postexpandincludesize":{"value":136196,"limit":2097152},"templateargumentsize":{"value":2283,"limit":2097152},"expansiondepth":{"value":16,"limit":100},"expensivefunctioncount":{"value":4,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":205713,"limit":5000000},"entityaccesscount":{"value":1,"limit":400},"timingprofile":["100.00% 787.113 1 -total"," 38.19% 300.579 1 Template:Reflist"," 22.63% 178.160 36 Template:Cite_journal"," 15.13% 119.115 9 Template:Cite_book"," 10.31% 81.122 1 Template:Information_theory"," 10.13% 79.732 1 Template:Sidebar"," 10.00% 78.705 1 Template:Short_description"," 7.56% 59.502 1 Template:Authority_control"," 7.19% 56.621 2 Template:Pagetype"," 5.54% 43.633 1 Template:Sfn"]},"scribunto":{"limitreport-timeusage":{"value":"0.510","limit":"10.000"},"limitreport-memusage":{"value":7994994,"limit":52428800},"limitreport-logs":"anchor_id_list = table#1 {\n [\"CITEREFAndre_S._RibeiroStuart_A._KauffmanJason_Lloyd-PriceBjorn_Samuelsson2008\"] = 1,\n [\"CITEREFArcherParkPillow2013\"] = 1,\n [\"CITEREFBaudotTapiaBennequinGoaillard2019\"] = 2,\n [\"CITEREFBrennerStrongKoberleBialek2000\"] = 1,\n [\"CITEREFChristopher_D._ManningPrabhakar_RaghavanHinrich_Schütze2008\"] = 1,\n [\"CITEREFChurchHanks1989\"] = 1,\n [\"CITEREFCilibrasiVitányi2005\"] = 1,\n [\"CITEREFCoombsDawesTversky1970\"] = 1,\n [\"CITEREFCoverThomas1991\"] = 1,\n [\"CITEREFCoverThomas2005\"] = 1,\n [\"CITEREFCronbach1954\"] = 1,\n [\"CITEREFEverett1957\"] = 1,\n [\"CITEREFGel\u0026#039;fandYaglom1957\"] = 1,\n [\"CITEREFGuiasu1977\"] = 1,\n [\"CITEREFHaghighatAghagolzadehSeyedarabi2011\"] = 2,\n [\"CITEREFHu1962\"] = 1,\n [\"CITEREFHutter2001\"] = 1,\n [\"CITEREFJanssenGuanRobeva2023\"] = 2,\n [\"CITEREFKeysKholikovPevtsov2015\"] = 1,\n [\"CITEREFKreer1957\"] = 1,\n [\"CITEREFKvålseth1991\"] = 1,\n [\"CITEREFLee2021\"] = 1,\n [\"CITEREFLiVitányi1997\"] = 1,\n [\"CITEREFLockhead1970\"] = 1,\n [\"CITEREFMassey1990\"] = 1,\n [\"CITEREFMcGill1954\"] = 1,\n [\"CITEREFNicolettiBusiello2021\"] = 1,\n [\"CITEREFNicolettiBusiello2022\"] = 1,\n [\"CITEREFNicolettiBusiello2024\"] = 1,\n [\"CITEREFPandeySarkar2017\"] = 1,\n [\"CITEREFPeng,_H.C.Long,_F.Ding,_C.2005\"] = 1,\n [\"CITEREFPermuterWeissmanGoldsmith2009\"] = 1,\n [\"CITEREFPocock2012\"] = 1,\n [\"CITEREFPressTeukolskyVetterlingFlannery2007\"] = 1,\n [\"CITEREFRajski1961\"] = 2,\n [\"CITEREFStrehlGhosh2003\"] = 1,\n [\"CITEREFTapiaBaudotFormizano-TrezinyDufour2018\"] = 1,\n [\"CITEREFTomasz_JetkaKarol_NienaltowskiTomasz_WinarskiSlawomir_Blonski2019\"] = 1,\n [\"CITEREFWatkinsonLiangWangZheng2009\"] = 1,\n [\"CITEREFWellsViolaAtsumiNakajima1996\"] = 1,\n [\"CITEREFWhiteSteingoldFournelle\"] = 1,\n [\"CITEREFWijayaSarnoZulaika2017\"] = 1,\n [\"CITEREFWittenFrank2005\"] = 1,\n [\"CITEREFWolpertDeDeo2013\"] = 1,\n [\"CITEREFWolpertWolf1995\"] = 1,\n}\ntemplate_list = table#1 {\n [\"Authority control\"] = 1,\n [\"CS1 config\"] = 1,\n [\"Citation\"] = 1,\n [\"Cite book\"] = 9,\n [\"Cite conference\"] = 2,\n [\"Cite journal\"] = 36,\n [\"Cite thesis\"] = 1,\n [\"Cite web\"] = 2,\n [\"Equation box 1\"] = 7,\n [\"EquationRef\"] = 2,\n [\"Google Code\"] = 1,\n [\"Harv\"] = 3,\n [\"Harvard citation\"] = 2,\n [\"Harvnb\"] = 3,\n [\"Information theory\"] = 1,\n [\"Isbn\"] = 1,\n [\"Main\"] = 3,\n [\"NumBlk\"] = 2,\n [\"Reflist\"] = 1,\n [\"Rp\"] = 3,\n [\"Sfn\"] = 1,\n [\"Short description\"] = 1,\n}\narticle_whitelist = table#1 {\n}\nciteref_patterns = table#1 {\n}\n"},"cachereport":{"origin":"mw-web.codfw.main-65b64b4b74-ktrd4","timestamp":"20250219122717","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Mutual information","url":"https:\/\/en.wikipedia.org\/wiki\/Mutual_information","sameAs":"http:\/\/www.wikidata.org\/entity\/Q252973","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q252973","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2004-01-11T09:47:33Z","dateModified":"2025-01-19T02:09:14Z","image":"https:\/\/upload.wikimedia.org\/wikipedia\/commons\/d\/d4\/Entropy-mutual-information-relative-entropy-relation-diagram.svg","headline":"measure of dependence between two variables"}</script> </body> </html>

CINXE.COM

Mutual information - Wikipedia