CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Permutation test - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"0b350c44-85ea-4a8c-b138-b60eb64be02d","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Permutation_test","wgTitle":"Permutation test","wgCurRevisionId":1255317817,"wgRevisionId":1255317817,"wgArticleId":2468117,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Articles with short description","Short description matches Wikidata","All articles with unsourced statements","Articles with unsourced statements from January 2020","Statistical tests","Nonparametric statistics"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Permutation_test","wgRelevantArticleId":2468117,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove": [],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":20000,"wgRelatedArticlesCompat":[],"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q108839445","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false, "wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.math.styles":"ready","ext.cite.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","mediawiki.page.media","ext.scribunto.logs","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin", "mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession","wikibase.sidebar.tracking"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Permutation test - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Permutation_test"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Permutation_test&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Permutation_test"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Permutation_test rootpage-Permutation_test skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia"><span>Contents</span></a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events"><span>Current events</span></a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x"><span>Random article</span></a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works"><span>About Wikipedia</span></a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia"><span>Contact us</span></a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia"><span>Help</span></a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia"><span>Learn to edit</span></a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors"><span>Community portal</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia"><span>Upload file</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Permutation+test" title="You are encouraged to create an account and log in; however, it is not mandatory" class=""><span>Create account</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Permutation+test" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Permutation+test" title="You are encouraged to create an account and log in; however, it is not mandatory"><span class="vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd"></span> <span>Create account</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Permutation+test" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing"><span>learn more</span></a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y"><span>Contributions</span></a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n"><span>Talk</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-Method" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Method"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Method</span> </div> </a> <ul id="toc-Method-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Relation_to_parametric_tests" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Relation_to_parametric_tests"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Relation to parametric tests</span> </div> </a> <ul id="toc-Relation_to_parametric_tests-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Advantages" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Advantages"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Advantages</span> </div> </a> <ul id="toc-Advantages-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Limitations" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Limitations"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Limitations</span> </div> </a> <ul id="toc-Limitations-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Monte_Carlo_testing" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Monte_Carlo_testing"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Monte Carlo testing</span> </div> </a> <ul id="toc-Monte_Carlo_testing-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Example_tests" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Example_tests"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>Example tests</span> </div> </a> <ul id="toc-Example_tests-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>See also</span> </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Literature" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Literature"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>Literature</span> </div> </a> <button aria-controls="toc-Literature-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Literature subsection</span> </button> <ul id="toc-Literature-sublist" class="vector-toc-list"> <li id="toc-Current_research_on_permutation_tests" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Current_research_on_permutation_tests"> <div class="vector-toc-text"> <span class="vector-toc-numb">8.1</span> <span>Current research on permutation tests</span> </div> </a> <ul id="toc-Current_research_on_permutation_tests-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Permutation test</span></h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 4 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-4" aria-hidden="true" ><span class="vector-icon mw-ui-icon-language-progressive mw-ui-icon-wikimedia-language-progressive"></span> <span class="vector-dropdown-label-text">4 languages</span> </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Permutationstest" title="Permutationstest – German" lang="de" hreflang="de" data-title="Permutationstest" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target"><span>Deutsch</span></a></li><li class="interlanguage-link interwiki-pl mw-list-item"><a href="https://pl.wikipedia.org/wiki/Test_permutacyjny" title="Test permutacyjny – Polish" lang="pl" hreflang="pl" data-title="Test permutacyjny" data-language-autonym="Polski" data-language-local-name="Polish" class="interlanguage-link-target"><span>Polski</span></a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%9F%D0%B5%D1%80%D0%BC%D1%83%D1%82%D0%B0%D1%86%D1%96%D0%B9%D0%BD%D0%B8%D0%B9_%D1%82%D0%B5%D1%81%D1%82" title="Пермутаційний тест – Ukrainian" lang="uk" hreflang="uk" data-title="Пермутаційний тест" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target"><span>Українська</span></a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E7%BD%AE%E6%8D%A2%E6%A3%80%E9%AA%8C" title="置换检验 – Chinese" lang="zh" hreflang="zh" data-title="置换检验" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target"><span>中文</span></a></li> </ul> <div class="after-portlet after-portlet-lang"><span class="wb-langlinks-edit wb-langlinks-link"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q108839445#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></span></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Permutation_test" title="View the content page [c]" accesskey="c"><span>Article</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Permutation_test" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t"><span>Talk</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Permutation_test"><span>Read</span></a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Permutation_test&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Permutation_test&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Permutation_test"><span>Read</span></a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Permutation_test&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Permutation_test&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Permutation_test" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Permutation_test" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u"><span>Upload file</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Permutation_test&oldid=1255317817" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Permutation_test&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Permutation_test&id=1255317817&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FPermutation_test"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FPermutation_test"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Permutation_test&action=show-download-screen" title="Download this page as a PDF file"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Permutation_test&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q108839445" title="Structured data on this page hosted by Wikidata [g]" accesskey="g"><span>Wikidata item</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Exact statistical hypothesis test</div> <p>A <b>permutation test</b> (also called re-randomization test or shuffle test) is an <a href="/wiki/Exact_test" title="Exact test">exact</a> <a href="/wiki/Statistical_hypothesis_testing" class="mw-redirect" title="Statistical hypothesis testing">statistical hypothesis test</a> making use of the <a href="/wiki/Proof_by_contradiction" title="Proof by contradiction">proof by contradiction</a>. A permutation test involves two or more samples. The null hypothesis is that all samples come from the same distribution <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H_{0}:F=G}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>H</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>:</mo> <mi>F</mi> <mo>=</mo> <mi>G</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle H_{0}:F=G}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9b587413f1bf05485516f44cd01072ad44cca27c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.589ex; height:2.509ex;" alt="{\displaystyle H_{0}:F=G}"></span>. Under the <a href="/wiki/Null_hypothesis" title="Null hypothesis">null hypothesis</a>, the distribution of the test statistic is obtained by calculating all possible values of the <a href="/wiki/Test_statistic" title="Test statistic">test statistic</a> under possible rearrangements of the observed data. Permutation tests are, therefore, a form of <a href="/wiki/Resampling_(statistics)" title="Resampling (statistics)">resampling</a>. </p><p>Permutation tests can be understood as <a href="/wiki/Surrogate_data_testing" title="Surrogate data testing">surrogate data testing</a> where the <a href="/wiki/Surrogate_data" title="Surrogate data">surrogate data</a> under the null hypothesis are obtained through permutations of the original data.<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> </p><p>In other words, the method by which treatments are allocated to subjects in an experimental design is mirrored in the analysis of that design. If the labels are exchangeable under the null hypothesis, then the resulting tests yield exact significance levels; see also <a href="/wiki/Exchangeability" class="mw-redirect" title="Exchangeability">exchangeability</a>. Confidence intervals can then be derived from the tests. The theory has evolved from the works of <a href="/wiki/Ronald_Fisher" title="Ronald Fisher">Ronald Fisher</a> and <a href="/wiki/E._J._G._Pitman" title="E. J. G. Pitman">E. J. G. Pitman</a> in the 1930s. </p><p>Permutation tests should not be confused with <a href="/wiki/Randomized_test" class="mw-redirect" title="Randomized test">randomized tests</a>.<sup id="cite_ref-2" class="reference"><a href="#cite_note-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p> <meta property="mw:PageProp/toc" /> <div class="mw-heading mw-heading2"><h2 id="Method">Method</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=1" title="Edit section: Method"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure typeof="mw:File/Thumb"><a href="/wiki/File:Permutation_test_example_animation.gif" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/1/12/Permutation_test_example_animation.gif/300px-Permutation_test_example_animation.gif" decoding="async" width="300" height="242" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/1/12/Permutation_test_example_animation.gif/450px-Permutation_test_example_animation.gif 1.5x, //upload.wikimedia.org/wikipedia/commons/1/12/Permutation_test_example_animation.gif 2x" data-file-width="519" data-file-height="419" /></a><figcaption>Animation of a permutation test being computed on sets of 4 and 5 random values. The 4 values in red are drawn from one distribution, and the 5 values in blue from another; we'd like to test whether the mean values of the two distributions are different. The hypothesis is that the mean of the first distribution is higher than the mean of the second; the null hypothesis is that both groups of samples are drawn from the same distribution. There are 126 distinct ways to put 4 values into one group and 5 into another (9-choose-4 or 9-choose-5). Of these, one is per the original labeling, and the other 125 are "permutations" that generate the histogram of mean differences <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {\mu }}_{1}-{\hat {\mu }}_{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>μ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> </mrow> </msub> <mo>−</mo> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>μ</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\hat {\mu }}_{1}-{\hat {\mu }}_{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bb9aae532f442268ce170189c8e41abfaf1ec06c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.752ex; height:2.676ex;" alt="{\displaystyle {\hat {\mu }}_{1}-{\hat {\mu }}_{2}}"></span> shown. The p-value of the hypothesis is estimated as the proportion of permutations that give a difference as large or larger than the difference of means of the original samples. In this example, the null hypothesis cannot be rejected at the <span class="texhtml"><i>p</i> = 5%</span> level.</figcaption></figure> <p>To illustrate the basic idea of a permutation test, suppose we collect random variables <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{A}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>A</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X_{A}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0c3b750756d33b2e60efd90c59a059d0dae129cd" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.389ex; height:2.509ex;" alt="{\displaystyle X_{A}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{B}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>B</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X_{B}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e93fabcf27b168d679d6469c36fe4c1d460b96d4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.404ex; height:2.509ex;" alt="{\displaystyle X_{B}}"></span> for each individual from two groups <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle A}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>A</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle A}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7daff47fa58cdfd29dc333def748ff5fa4c923e3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.743ex; height:2.176ex;" alt="{\displaystyle A}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>B</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle B}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/47136aad860d145f75f3eed3022df827cee94d7a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.764ex; height:2.176ex;" alt="{\displaystyle B}"></span> whose sample means are <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {x}}_{A}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>x</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>A</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {x}}_{A}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3601333d996a40a783be2c9772ddedb9e58b7fb3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.795ex; height:2.343ex;" alt="{\displaystyle {\bar {x}}_{A}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {x}}_{B}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>x</mi> <mo stretchy="false">¯</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>B</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\bar {x}}_{B}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3f7142baa532e0ba856b662e4b9003f2f1f458d8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.809ex; height:2.343ex;" alt="{\displaystyle {\bar {x}}_{B}}"></span>, and that we want to know whether <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{A}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>A</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X_{A}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0c3b750756d33b2e60efd90c59a059d0dae129cd" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.389ex; height:2.509ex;" alt="{\displaystyle X_{A}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X_{B}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>X</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>B</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle X_{B}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/e93fabcf27b168d679d6469c36fe4c1d460b96d4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.404ex; height:2.509ex;" alt="{\displaystyle X_{B}}"></span> come from the same distribution. Let <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{A}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>A</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n_{A}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ae6767464c4260a2d766b108cba5d3028a82c25d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.86ex; height:2.009ex;" alt="{\displaystyle n_{A}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{B}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>B</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n_{B}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/722dd99cf487a0afe2a1020549fc19a31eec3424" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.874ex; height:2.009ex;" alt="{\displaystyle n_{B}}"></span> be the sample size collected from each group. The permutation test is designed to determine whether the observed difference between the sample means is large enough to reject, at some significance level, the null hypothesis H<span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle _{0}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi></mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle _{0}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/bd058dac6302ece0ec9a3e1cfad8a36ca674903a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.054ex; height:1.676ex;" alt="{\displaystyle _{0}}"></span> that the data drawn from <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle A}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>A</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle A}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7daff47fa58cdfd29dc333def748ff5fa4c923e3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.743ex; height:2.176ex;" alt="{\displaystyle A}"></span> is from the same distribution as the data drawn from <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>B</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle B}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/47136aad860d145f75f3eed3022df827cee94d7a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.764ex; height:2.176ex;" alt="{\displaystyle B}"></span>. </p><p>The test proceeds as follows. First, the difference in means between the two samples is calculated: this is the observed value of the test statistic, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T_{\text{obs}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>T</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>obs</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T_{\text{obs}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1b5d7b67b0a18d8fcfc605c9de30e926c77c722a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.974ex; height:2.509ex;" alt="{\displaystyle T_{\text{obs}}}"></span>. </p><p>Next, the observations of groups <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle A}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>A</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle A}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7daff47fa58cdfd29dc333def748ff5fa4c923e3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.743ex; height:2.176ex;" alt="{\displaystyle A}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>B</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle B}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/47136aad860d145f75f3eed3022df827cee94d7a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.764ex; height:2.176ex;" alt="{\displaystyle B}"></span> are pooled, and the difference in sample means is calculated and recorded for every possible way of dividing the pooled values into two groups of size <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{A}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>A</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n_{A}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ae6767464c4260a2d766b108cba5d3028a82c25d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.86ex; height:2.009ex;" alt="{\displaystyle n_{A}}"></span> and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{B}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>B</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n_{B}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/722dd99cf487a0afe2a1020549fc19a31eec3424" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.874ex; height:2.009ex;" alt="{\displaystyle n_{B}}"></span> (i.e., for every permutation of the group labels A and B). The set of these calculated differences is the exact distribution of possible differences (for this sample) under the null hypothesis that group labels are exchangeable (i.e., are randomly assigned). </p><p>The one-sided p-value of the test is calculated as the proportion of sampled permutations where the difference in means was greater than <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T_{\text{obs}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>T</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>obs</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T_{\text{obs}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1b5d7b67b0a18d8fcfc605c9de30e926c77c722a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.974ex; height:2.509ex;" alt="{\displaystyle T_{\text{obs}}}"></span>. The two-sided p-value of the test is calculated as the proportion of sampled permutations where the <a href="/wiki/Absolute_difference" title="Absolute difference">absolute difference</a> was greater than <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle |T_{\text{obs}}|}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> <msub> <mi>T</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>obs</mtext> </mrow> </msub> <mrow class="MJX-TeXAtom-ORD"> <mo stretchy="false">|</mo> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle |T_{\text{obs}}|}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/5a0124570dbe8c399fe175b970af3157023b397d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.267ex; height:2.843ex;" alt="{\displaystyle |T_{\text{obs}}|}"></span>. Many implementations of permutation tests require that the observed data itself be counted as one of the permutations so that the permutation p-value will never be zero.<sup id="cite_ref-3" class="reference"><a href="#cite_note-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> </p><p>Alternatively, if the only purpose of the test is to reject or not reject the null hypothesis, one could sort the recorded differences, and then observe if <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T_{\text{obs}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>T</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>obs</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T_{\text{obs}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1b5d7b67b0a18d8fcfc605c9de30e926c77c722a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.974ex; height:2.509ex;" alt="{\displaystyle T_{\text{obs}}}"></span> is contained within the middle <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (1-\alpha )\times 100}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <mi>α</mi> <mo stretchy="false">)</mo> <mo>×</mo> <mn>100</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle (1-\alpha )\times 100}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/48e2df60c5c88abb7eae62c9d6a32c7a6e43fb07" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.628ex; height:2.843ex;" alt="{\displaystyle (1-\alpha )\times 100}"></span>% of them, for some significance level <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b79333175c8b3f0840bfb4ec41b8072c83ea88d3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.488ex; height:1.676ex;" alt="{\displaystyle \alpha }"></span>. If it is not, we reject the hypothesis of identical probability curves at the <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha \times 100\%}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>α</mi> <mo>×</mo> <mn>100</mn> <mi mathvariant="normal">%</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \alpha \times 100\%}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0b72fffc9e71af2b22609b2023761aed2ded919b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:9.751ex; height:2.343ex;" alt="{\displaystyle \alpha \times 100\%}"></span> significance level. </p><p>To exploit <a href="/wiki/Variance_reduction" title="Variance reduction">variance reduction</a> with <a href="/wiki/Paired_sample" class="mw-redirect" title="Paired sample">paired samples</a> the paired permutation test needs to be applied, see <a href="/wiki/Paired_difference_test" title="Paired difference test">paired difference test</a>. </p> <div class="mw-heading mw-heading2"><h2 id="Relation_to_parametric_tests">Relation to parametric tests</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=2" title="Edit section: Relation to parametric tests"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Permutation tests are a subset of <a href="/wiki/Non-parametric_statistics" class="mw-redirect" title="Non-parametric statistics">non-parametric statistics</a>. Assuming that our experimental data come from data measured from two treatment groups, the method simply generates the distribution of mean differences under the assumption that the two groups are not distinct in terms of the measured variable. From this, one then uses the observed statistic (<span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T_{\text{obs}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>T</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>obs</mtext> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle T_{\text{obs}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1b5d7b67b0a18d8fcfc605c9de30e926c77c722a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.974ex; height:2.509ex;" alt="{\displaystyle T_{\text{obs}}}"></span> above) to see to what extent this statistic is special, i.e., the likelihood of observing the magnitude of such a value (or larger) if the treatment labels had simply been randomized after treatment. </p><p>In contrast to permutation tests, the distributions underlying many popular <a href="/wiki/Classical_statistics" class="mw-redirect" title="Classical statistics">"classical" statistical</a> tests, such as the <a href="/wiki/T-test" class="mw-redirect" title="T-test"><i>t</i>-test</a>, <a href="/wiki/F-test" title="F-test"><i>F</i>-test</a>, <a href="/wiki/Z-test" title="Z-test"><i>z</i>-test</a>, and <a href="/wiki/Chi-squared_test" title="Chi-squared test"><i>χ</i><sup>2</sup> test</a>, are obtained from theoretical <a href="/wiki/Probability_distribution" title="Probability distribution">probability distributions</a>. <a href="/wiki/Fisher%27s_exact_test" title="Fisher's exact test">Fisher's exact test</a> is an example of a commonly used parametric test for evaluating the association between two dichotomous variables. When sample sizes are very large, the Pearson's chi-square test will give accurate results. For small samples, the chi-square reference distribution cannot be assumed to give a correct description of the probability distribution of the test statistic, and in this situation the use of Fisher's exact test becomes more appropriate. </p><p>Permutation tests exist in many situations where parametric tests do not (e.g., when deriving an optimal test when losses are proportional to the size of an error rather than its square). All simple and many relatively complex parametric tests have a corresponding permutation test version that is defined by using the same test statistic as the parametric test, but obtains the p-value from the sample-specific permutation distribution of that statistic, rather than from the theoretical distribution derived from the parametric assumption. For example, it is possible in this manner to construct a permutation <a href="/wiki/T-test" class="mw-redirect" title="T-test"><i>t</i>-test</a>, a permutation <a href="/wiki/Chi-squared_test" title="Chi-squared test"><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \chi ^{2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <msup> <mi>χ</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mstyle> </mrow> <annotation encoding="application/x-tex">{\textstyle \chi ^{2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c300b17568b2d26b926c6d83699ae7431b318009" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.509ex; height:2.843ex;" alt="{\textstyle \chi ^{2}}"></span> test</a> of association, a permutation version of Aly's test for comparing variances and so on. </p><p>The major drawbacks to permutation tests are that they </p> <ul><li>Can be computationally intensive and may require "custom" code for difficult-to-calculate statistics. This must be rewritten for every case.</li> <li>Are primarily used to provide a p-value. The inversion of the test to get confidence regions/intervals requires even more computation.</li></ul> <div class="mw-heading mw-heading2"><h2 id="Advantages">Advantages</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=3" title="Edit section: Advantages"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Permutation tests exist for any test statistic, regardless of whether or not its distribution is known. Thus one is always free to choose the statistic which best discriminates between hypothesis and alternative and which minimizes losses. </p><p>Permutation tests can be used for analyzing unbalanced designs<sup id="cite_ref-4" class="reference"><a href="#cite_note-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup> and for combining dependent tests on mixtures of categorical, ordinal, and metric data (Pesarin, 2001) <sup class="noprint Inline-Template Template-Fact" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed"><span title="This claim needs references to reliable sources. (January 2020)">citation needed</span></a></i>]</sup>. They can also be used to analyze qualitative data that has been quantitized (i.e., turned into numbers). Permutation tests may be ideal for analyzing quantitized data that do not satisfy statistical assumptions underlying traditional parametric tests (e.g., t-tests, ANOVA),<sup id="cite_ref-5" class="reference"><a href="#cite_note-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> see <a href="/wiki/PERMANOVA" class="mw-redirect" title="PERMANOVA">PERMANOVA</a>. </p><p>Before the 1980s, the burden of creating the reference distribution was overwhelming except for data sets with small sample sizes. </p><p>Since the 1980s, the confluence of relatively inexpensive fast computers and the development of new sophisticated path algorithms applicable in special situations made the application of permutation test methods practical for a wide range of problems. It also initiated the addition of exact-test options in the main statistical software packages and the appearance of specialized software for performing a wide range of uni- and multi-variable exact tests and computing test-based "exact" confidence intervals. </p> <div class="mw-heading mw-heading2"><h2 id="Limitations">Limitations</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=4" title="Edit section: Limitations"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>An important assumption behind a permutation test is that the observations are exchangeable under the null hypothesis. An important consequence of this assumption is that tests of difference in location (like a permutation t-test) require equal variance under the normality assumption. In this respect, the classic permutation t-test shares the same weakness as the classical Student's t-test (the <a href="/wiki/Behrens%E2%80%93Fisher_problem" title="Behrens–Fisher problem">Behrens–Fisher problem</a>). This can be addressed in the same way the classic t-test has been extended to handle unequal variances: by employing the <a href="/w/index.php?title=Welch_statistic&action=edit&redlink=1" class="new" title="Welch statistic (page does not exist)">Welch statistic</a> with Satterthwaite adjustment to the degrees of freedom.<sup id="cite_ref-6" class="reference"><a href="#cite_note-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> A third alternative in this situation is to use a <a href="/w/index.php?title=Bootstrap-based_test&action=edit&redlink=1" class="new" title="Bootstrap-based test (page does not exist)">bootstrap-based test</a>. Statistician <a href="/wiki/Phillip_Good" title="Phillip Good">Phillip Good</a> explains the difference between permutation tests and bootstrap tests the following way: "Permutations test hypotheses concerning distributions; bootstraps test hypotheses concerning parameters. As a result, the bootstrap entails less-stringent assumptions."<sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup> Bootstrap tests are not exact. In some cases, a permutation test based on a properly studentized statistic can be asymptotically exact even when the exchangeability assumption is violated.<sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> Bootstrap-based tests can test with the null hypothesis <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle H_{0}:F\neq G}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>H</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>0</mn> </mrow> </msub> <mo>:</mo> <mi>F</mi> <mo>≠</mo> <mi>G</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle H_{0}:F\neq G}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fceb6ecc6593234232b2bff77805f2931a4faa49" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.589ex; height:2.676ex;" alt="{\displaystyle H_{0}:F\neq G}"></span> and, therefore, are suited for performing <a href="/wiki/Equivalence_test" title="Equivalence test">equivalence testing</a>. </p> <div class="mw-heading mw-heading2"><h2 id="Monte_Carlo_testing">Monte Carlo testing</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=5" title="Edit section: Monte Carlo testing"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>An asymptotically equivalent permutation test can be created when there are too many possible orderings of the data to allow complete enumeration in a convenient manner. This is done by generating the reference distribution by <a href="/wiki/Monte_Carlo_sampling" class="mw-redirect" title="Monte Carlo sampling">Monte Carlo sampling</a>, which takes a small (relative to the total number of permutations) random sample of the possible replicates. The realization that this could be applied to any permutation test on any dataset was an important breakthrough in the area of applied statistics. The earliest known references to this approach are Eden and <a href="/wiki/Frank_Yates" title="Frank Yates">Yates</a> (1933) and <a href="/wiki/Meyer_Dwass" title="Meyer Dwass">Dwass</a> (1957).<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-10" class="reference"><a href="#cite_note-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> This type of permutation test is known under various names: <i>approximate permutation test</i>, <i>Monte Carlo permutation tests</i> or <i>random permutation tests</i>.<sup id="cite_ref-11" class="reference"><a href="#cite_note-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> </p><p>After <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>N</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle N}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f5e3890c981ae85503089652feb48b191b57aae3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.064ex; height:2.176ex;" alt="{\displaystyle N}"></span> random permutations, it is possible to obtain a confidence interval for the p-value based on the Binomial distribution, see <a href="/wiki/Binomial_proportion_confidence_interval" title="Binomial proportion confidence interval">Binomial proportion confidence interval</a>. For example, if after <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N=10000}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>N</mi> <mo>=</mo> <mn>10000</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle N=10000}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/8f13b17e1a7fbe046ab619ccc5167809d04872c2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:10.974ex; height:2.176ex;" alt="{\displaystyle N=10000}"></span> random permutations the p-value is estimated to be <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\widehat {p}}=0.05}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>p</mi> <mo>^</mo> </mover> </mrow> </mrow> <mo>=</mo> <mn>0.05</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\widehat {p}}=0.05}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a3263c217c4acb02916e58d21103d20850f7d5c9" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:8.682ex; height:2.509ex;" alt="{\displaystyle {\widehat {p}}=0.05}"></span>, then a 99% confidence interval for the true <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/81eac1e205430d1f40810df36a0edffdc367af36" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:1.259ex; height:2.009ex;" alt="{\displaystyle p}"></span> (the one that would result from trying all possible permutations) is <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \left[{\hat {p}}-z{\sqrt {\frac {0.05(1-0.05)}{10000}}},{\hat {p}}+z{\sqrt {\frac {0.05(1-0.05)}{10000}}}\right]=[0.045,0.055]}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow> <mo>[</mo> <mrow> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>p</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mo>−</mo> <mi>z</mi> <mrow class="MJX-TeXAtom-ORD"> <msqrt> <mfrac> <mrow> <mn>0.05</mn> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <mn>0.05</mn> <mo stretchy="false">)</mo> </mrow> <mn>10000</mn> </mfrac> </msqrt> </mrow> <mo>,</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>p</mi> <mo stretchy="false">^</mo> </mover> </mrow> </mrow> <mo>+</mo> <mi>z</mi> <mrow class="MJX-TeXAtom-ORD"> <msqrt> <mfrac> <mrow> <mn>0.05</mn> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <mn>0.05</mn> <mo stretchy="false">)</mo> </mrow> <mn>10000</mn> </mfrac> </msqrt> </mrow> </mrow> <mo>]</mo> </mrow> <mo>=</mo> <mo stretchy="false">[</mo> <mn>0.045</mn> <mo>,</mo> <mn>0.055</mn> <mo stretchy="false">]</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \left[{\hat {p}}-z{\sqrt {\frac {0.05(1-0.05)}{10000}}},{\hat {p}}+z{\sqrt {\frac {0.05(1-0.05)}{10000}}}\right]=[0.045,0.055]}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/add2b4f210edaca5450abe5ec9b6199fb5af7d8f" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.671ex; width:65.211ex; height:8.509ex;" alt="{\displaystyle \left[{\hat {p}}-z{\sqrt {\frac {0.05(1-0.05)}{10000}}},{\hat {p}}+z{\sqrt {\frac {0.05(1-0.05)}{10000}}}\right]=[0.045,0.055]}"></span>. </p><p>On the other hand, the purpose of estimating the p-value is most often to decide whether <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p\leq \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo>≤</mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p\leq \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d5bae8c2feb17e55ae0aadd1e7b541a7afba931" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:5.845ex; height:2.343ex;" alt="{\displaystyle p\leq \alpha }"></span>, where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \scriptstyle \ \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mstyle displaystyle="false" scriptlevel="1"> <mtext> </mtext> <mi>α</mi> </mstyle> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \scriptstyle \ \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4d178e88ad1656331a273a7724f6236e2cb637c5" class="mwe-math-fallback-image-inline mw-invert skin-invert" style="vertical-align: -0.338ex; width:1.633ex; height:1.343ex;" aria-hidden="true" alt="{\displaystyle \scriptstyle \ \alpha }"></span> is the threshold at which the null hypothesis will be rejected (typically <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha =0.05}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>α</mi> <mo>=</mo> <mn>0.05</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \alpha =0.05}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9fdb2e1d36bb136f99c1a76e11e8bc05906354b8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:8.72ex; height:2.176ex;" alt="{\displaystyle \alpha =0.05}"></span>). In the example above, the confidence interval only tells us that there is roughly a 50% chance that the p-value is smaller than 0.05, i.e. it is completely unclear whether the null hypothesis should be rejected at a level <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha =0.05}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>α</mi> <mo>=</mo> <mn>0.05</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \alpha =0.05}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/9fdb2e1d36bb136f99c1a76e11e8bc05906354b8" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:8.72ex; height:2.176ex;" alt="{\displaystyle \alpha =0.05}"></span>. </p><p>If it is only important to know whether <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p\leq \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo>≤</mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p\leq \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d5bae8c2feb17e55ae0aadd1e7b541a7afba931" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:5.845ex; height:2.343ex;" alt="{\displaystyle p\leq \alpha }"></span> for a given <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b79333175c8b3f0840bfb4ec41b8072c83ea88d3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.488ex; height:1.676ex;" alt="{\displaystyle \alpha }"></span>, it is logical to continue simulating until the statement <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p\leq \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo>≤</mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p\leq \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d5bae8c2feb17e55ae0aadd1e7b541a7afba931" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:5.845ex; height:2.343ex;" alt="{\displaystyle p\leq \alpha }"></span> can be established to be true or false with a very low probability of error. Given a bound <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \epsilon }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>ϵ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \epsilon }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c3837cad72483d97bcdde49c85d3b7b859fb3fd2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.944ex; height:1.676ex;" alt="{\displaystyle \epsilon }"></span> on the admissible probability of error (the probability of finding that <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\widehat {p}}>\alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <mi>p</mi> <mo>^</mo> </mover> </mrow> </mrow> <mo>></mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\widehat {p}}>\alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a8352e351fe8de49d36463f09c80ab88767a2153" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:6.035ex; height:2.509ex;" alt="{\displaystyle {\widehat {p}}>\alpha }"></span> when in fact <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p\leq \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo>≤</mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p\leq \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d5bae8c2feb17e55ae0aadd1e7b541a7afba931" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:5.845ex; height:2.343ex;" alt="{\displaystyle p\leq \alpha }"></span> or vice versa), the question of how many permutations to generate can be seen as the question of when to stop generating permutations, based on the outcomes of the simulations so far, in order to guarantee that the conclusion (which is either <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p\leq \alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo>≤</mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p\leq \alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6d5bae8c2feb17e55ae0aadd1e7b541a7afba931" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:5.845ex; height:2.343ex;" alt="{\displaystyle p\leq \alpha }"></span> or <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p>\alpha }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>p</mi> <mo>></mo> <mi>α</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p>\alpha }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/4146efa9fa314c8380b44dd655f2fb89f9ae03b3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:5.845ex; height:2.176ex;" alt="{\displaystyle p>\alpha }"></span>) is correct with probability at least as large as <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 1-\epsilon }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mn>1</mn> <mo>−</mo> <mi>ϵ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle 1-\epsilon }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/57f9b07affe80ff61cdc4f2e47977c8421a59c73" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:4.947ex; height:2.343ex;" alt="{\displaystyle 1-\epsilon }"></span>. (<span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \epsilon }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>ϵ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \epsilon }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c3837cad72483d97bcdde49c85d3b7b859fb3fd2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.944ex; height:1.676ex;" alt="{\displaystyle \epsilon }"></span> will typically be chosen to be extremely small, e.g. 1/1000.) Stopping rules to achieve this have been developed<sup id="cite_ref-12" class="reference"><a href="#cite_note-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> which can be incorporated with minimal additional computational cost. In fact, depending on the true underlying p-value it will often be found that the number of simulations required is remarkably small (e.g. as low as 5 and often not larger than 100) before a decision can be reached with virtual certainty. </p> <div class="mw-heading mw-heading2"><h2 id="Example_tests">Example tests</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=6" title="Edit section: Example tests"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Permutational_analysis_of_variance" title="Permutational analysis of variance">Permutational analysis of variance</a></li> <li><a href="/wiki/Rank_product" title="Rank product">Rank product</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=7" title="Edit section: See also"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Rank_test" title="Rank test">Rank test</a></li> <li><a href="/wiki/Lepage_test" title="Lepage test">Lepage test</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="Literature">Literature</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=8" title="Edit section: Literature"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Original references: </p> <ul><li><a href="/wiki/R._A._Fisher" class="mw-redirect" title="R. A. Fisher">Fisher, R.A.</a> (1935) <i><a href="/wiki/The_Design_of_Experiments" title="The Design of Experiments">The Design of Experiments</a></i>, New York: <a href="/w/index.php?title=Hafner_Publishing&action=edit&redlink=1" class="new" title="Hafner Publishing (page does not exist)">Hafner</a></li> <li><a href="/wiki/E._J._G._Pitman" title="E. J. G. Pitman">Pitman, E. J. G.</a> (1937) "Significance tests which may be applied to samples from any population", <i>Royal Statistical Society Supplement</i>, 4: 119-130 and 225-32 (parts I and II). <a href="/wiki/JSTOR_(identifier)" class="mw-redirect" title="JSTOR (identifier)">JSTOR</a> <a rel="nofollow" class="external text" href="https://www.jstor.org/stable/2984124">2984124</a> <a href="/wiki/JSTOR_(identifier)" class="mw-redirect" title="JSTOR (identifier)">JSTOR</a> <a rel="nofollow" class="external text" href="https://www.jstor.org/stable/2983647">2983647</a></li> <li><style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite id="CITEREFPitman1938" class="citation journal cs1"><a href="/wiki/E._J._G._Pitman" title="E. J. G. Pitman">Pitman, E. J. G.</a> (1938). "Significance tests which may be applied to samples from any population. Part III. The analysis of variance test". <i><a href="/wiki/Biometrika" title="Biometrika">Biometrika</a></i>. <b>29</b> (3–4): 322–335. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fbiomet%2F29.3-4.322">10.1093/biomet/29.3-4.322</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Biometrika&rft.atitle=Significance+tests+which+may+be+applied+to+samples+from+any+population.+Part+III.+The+analysis+of+variance+test&rft.volume=29&rft.issue=3%E2%80%934&rft.pages=322-335&rft.date=1938&rft_id=info%3Adoi%2F10.1093%2Fbiomet%2F29.3-4.322&rft.aulast=Pitman&rft.aufirst=E.+J.+G.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li></ul> <p>Modern references: </p> <ul><li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCollingridge2013" class="citation journal cs1">Collingridge, D.S. (2013). "A Primer on Quantitized Data Analysis and Permutation Testing". <i><a href="/wiki/Journal_of_Mixed_Methods_Research" title="Journal of Mixed Methods Research">Journal of Mixed Methods Research</a></i>. <b>7</b> (1): 79–95. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F1558689812454457">10.1177/1558689812454457</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:124618343">124618343</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Mixed+Methods+Research&rft.atitle=A+Primer+on+Quantitized+Data+Analysis+and+Permutation+Testing&rft.volume=7&rft.issue=1&rft.pages=79-95&rft.date=2013&rft_id=info%3Adoi%2F10.1177%2F1558689812454457&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A124618343%23id-name%3DS2CID&rft.aulast=Collingridge&rft.aufirst=D.S.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li> <li>Edgington, E. S., & Onghena, P. (2007) <i>Randomization tests</i>, 4th ed. New York: <a href="/w/index.php?title=Chapman_and_Hall/CRC&action=edit&redlink=1" class="new" title="Chapman and Hall/CRC (page does not exist)">Chapman and Hall/CRC</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9780367577711" title="Special:BookSources/9780367577711">9780367577711</a></li> <li>Good, Phillip I. (2005) <i>Permutation, Parametric and Bootstrap Tests of Hypotheses</i>, 3rd ed., <a href="/wiki/Springer_Science%2BBusiness_Media" title="Springer Science+Business Media">Springer</a> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/0-387-98898-X" title="Special:BookSources/0-387-98898-X">0-387-98898-X</a></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGood2002" class="citation journal cs1">Good, P (2002). <a rel="nofollow" class="external text" href="https://doi.org/10.22237%2Fjmasm%2F1036110240">"Extensions of the concept of exchangeability and their applications"</a>. <i><a href="/wiki/Journal_of_Modern_Applied_Statistical_Methods" title="Journal of Modern Applied Statistical Methods">Journal of Modern Applied Statistical Methods</a></i>. <b>1</b> (2): 243–247. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.22237%2Fjmasm%2F1036110240">10.22237/jmasm/1036110240</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Modern+Applied+Statistical+Methods&rft.atitle=Extensions+of+the+concept+of+exchangeability+and+their+applications&rft.volume=1&rft.issue=2&rft.pages=243-247&rft.date=2002&rft_id=info%3Adoi%2F10.22237%2Fjmasm%2F1036110240&rft.aulast=Good&rft.aufirst=P&rft_id=https%3A%2F%2Fdoi.org%2F10.22237%252Fjmasm%252F1036110240&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li> <li>Lunneborg, Cliff. (1999) <i>Data Analysis by Resampling</i>, Duxbury Press. <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/0-534-22110-6" title="Special:BookSources/0-534-22110-6">0-534-22110-6</a>.</li> <li>Pesarin, F. (2001). <i>Multivariate Permutation Tests : With Applications in Biostatistics</i>, <a href="/wiki/John_Wiley_%26_Sons" class="mw-redirect" title="John Wiley & Sons">John Wiley & Sons</a>. <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0471496700" title="Special:BookSources/978-0471496700">978-0471496700</a></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWelch1990" class="citation journal cs1">Welch, W. J. (1990). "Construction of permutation tests". <i><a href="/wiki/Journal_of_the_American_Statistical_Association" title="Journal of the American Statistical Association">Journal of the American Statistical Association</a></i>. <b>85</b> (411): 693–698. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F01621459.1990.10474929">10.1080/01621459.1990.10474929</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+American+Statistical+Association&rft.atitle=Construction+of+permutation+tests&rft.volume=85&rft.issue=411&rft.pages=693-698&rft.date=1990&rft_id=info%3Adoi%2F10.1080%2F01621459.1990.10474929&rft.aulast=Welch&rft.aufirst=W.+J.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li></ul> <p>Computational methods: </p> <ul><li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMehtaPatel1983" class="citation journal cs1">Mehta, C. R.; Patel, N. R. (1983). "A network algorithm for performing Fisher's exact test in r x c contingency tables". <i><a href="/wiki/Journal_of_the_American_Statistical_Association" title="Journal of the American Statistical Association">Journal of the American Statistical Association</a></i>. <b>78</b> (382): 427–434. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F01621459.1983.10477989">10.1080/01621459.1983.10477989</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+American+Statistical+Association&rft.atitle=A+network+algorithm+for+performing+Fisher%27s+exact+test+in+r+x+c+contingency+tables&rft.volume=78&rft.issue=382&rft.pages=427-434&rft.date=1983&rft_id=info%3Adoi%2F10.1080%2F01621459.1983.10477989&rft.aulast=Mehta&rft.aufirst=C.+R.&rft.au=Patel%2C+N.+R.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMehtaPatelSenchaudhuri1988" class="citation journal cs1">Mehta, C. R.; Patel, N. R.; Senchaudhuri, P. (1988). "Importance sampling for estimating exact probabilities in permutational inference". <i><a href="/wiki/Journal_of_the_American_Statistical_Association" title="Journal of the American Statistical Association">Journal of the American Statistical Association</a></i>. <b>83</b> (404): 999–1005. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F01621459.1988.10478691">10.1080/01621459.1988.10478691</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+American+Statistical+Association&rft.atitle=Importance+sampling+for+estimating+exact+probabilities+in+permutational+inference&rft.volume=83&rft.issue=404&rft.pages=999-1005&rft.date=1988&rft_id=info%3Adoi%2F10.1080%2F01621459.1988.10478691&rft.aulast=Mehta&rft.aufirst=C.+R.&rft.au=Patel%2C+N.+R.&rft.au=Senchaudhuri%2C+P.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGill2007" class="citation journal cs1">Gill, P. M. W. (2007). <a rel="nofollow" class="external text" href="http://rsc.anu.edu.au/%7Epgill/papers/103Fisher.pdf">"Efficient calculation of p-values in linear-statistic permutation significance tests"</a> <span class="cs1-format">(PDF)</span>. <i><a href="/wiki/Journal_of_Statistical_Computation_and_Simulation" title="Journal of Statistical Computation and Simulation">Journal of Statistical Computation and Simulation</a></i>. <b>77</b> (1): 55–61. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.708.1957">10.1.1.708.1957</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F10629360500108053">10.1080/10629360500108053</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:1813706">1813706</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Statistical+Computation+and+Simulation&rft.atitle=Efficient+calculation+of+p-values+in+linear-statistic+permutation+significance+tests&rft.volume=77&rft.issue=1&rft.pages=55-61&rft.date=2007&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.708.1957%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A1813706%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1080%2F10629360500108053&rft.aulast=Gill&rft.aufirst=P.+M.+W.&rft_id=http%3A%2F%2Frsc.anu.edu.au%2F%257Epgill%2Fpapers%2F103Fisher.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></li></ul> <div class="mw-heading mw-heading3"><h3 id="Current_research_on_permutation_tests">Current research on permutation tests</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=9" title="Edit section: Current research on permutation tests"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li>Good, P.I. (2012) Practitioner's Guide to Resampling Methods.</li> <li>Good, P.I. (2005) Permutation, Parametric, and Bootstrap Tests of Hypotheses</li> <li>Hesterberg, T. C., D. S. Moore, S. Monaghan, A. Clipson, and R. Epstein (2005): <a rel="nofollow" class="external text" href="https://web.archive.org/web/20060215221403/http://bcs.whfreeman.com/ips5e/content/cat_080/pdf/moore14.pdf">Bootstrap Methods and Permutation Tests</a>, <a rel="nofollow" class="external text" href="https://web.archive.org/web/20060110182635/http://www.insightful.com/Hesterberg/bootstrap/">software</a>.</li> <li>Moore, D. S., G. McCabe, W. Duckworth, and S. Sclove (2003): <a rel="nofollow" class="external text" href="https://statweb.stanford.edu/~tibs/stat315a/Supplements/bootstrap.pdf">Bootstrap Methods and Permutation Tests</a></li> <li>Simon, J. L. (1997): <a rel="nofollow" class="external text" href="https://web.archive.org/web/20051223034539/http://www.resample.com/content/text/index.shtml">Resampling: The New Statistics</a>.</li> <li>Yu, Chong Ho (2003): <a rel="nofollow" class="external text" href="http://PAREonline.net/getvn.asp?v=8&n=19">Resampling methods: concepts, applications, and justification. Practical Assessment, Research & Evaluation, 8(19)</a>. <i>(statistical bootstrapping)</i></li> <li><a rel="nofollow" class="external text" href="http://www.ericdigests.org/1993/marriage.htm">Resampling: A Marriage of Computers and Statistics (ERIC Digests)</a></li> <li>Pesarin, F., Salmaso, L. (2010). Permutation Tests for Complex Data: Theory, Applications and Software. Wiley. <a rel="nofollow" class="external free" href="https://books.google.com/books?id=9PWVTOanxPUC&hl=de">https://books.google.com/books?id=9PWVTOanxPUC&hl=de</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="References">References</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Permutation_test&action=edit&section=10" title="Edit section: References"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-references-wrap mw-references-columns"><ol class="references"> <li id="cite_note-1"><span class="mw-cite-backlink"><b><a href="#cite_ref-1">^</a></b></span> <span class="reference-text">Moore, Jason H. "Bootstrapping, permutation testing and the method of surrogate data." Physics in Medicine & Biology 44.6 (1999): L11.</span> </li> <li id="cite_note-2"><span class="mw-cite-backlink"><b><a href="#cite_ref-2">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFOnghena2017" class="citation cs2">Onghena, Patrick (2017-10-30), Berger, Vance W. (ed.), <a rel="nofollow" class="external text" href="https://www.taylorfrancis.com/books/9781315305103/chapters/10.1201/9781315305110-14">"Randomization Tests or Permutation Tests? A Historical and Terminological Clarification"</a>, <i>Randomization, Masking, and Allocation Concealment</i> (1 ed.), Boca Raton, FL: Chapman and Hall/CRC, pp. 209–228, <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1201%2F9781315305110-14">10.1201/9781315305110-14</a>, <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1-315-30511-0" title="Special:BookSources/978-1-315-30511-0"><bdi>978-1-315-30511-0</bdi></a><span class="reference-accessdate">, retrieved <span class="nowrap">2021-10-08</span></span></cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Randomization%2C+Masking%2C+and+Allocation+Concealment&rft.atitle=Randomization+Tests+or+Permutation+Tests%3F+A+Historical+and+Terminological+Clarification&rft.pages=209-228&rft.date=2017-10-30&rft_id=info%3Adoi%2F10.1201%2F9781315305110-14&rft.isbn=978-1-315-30511-0&rft.aulast=Onghena&rft.aufirst=Patrick&rft_id=https%3A%2F%2Fwww.taylorfrancis.com%2Fbooks%2F9781315305103%2Fchapters%2F10.1201%2F9781315305110-14&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-3"><span class="mw-cite-backlink"><b><a href="#cite_ref-3">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPhipsonSmyth2010" class="citation journal cs1">Phipson, Belinda; Smyth, Gordon K (2010). "Permutation p-values should never be zero: calculating exact p-values when permutations are randomly drawn". <i><a href="/wiki/Statistical_Applications_in_Genetics_and_Molecular_Biology" title="Statistical Applications in Genetics and Molecular Biology">Statistical Applications in Genetics and Molecular Biology</a></i>. <b>9</b> (1): Article 39. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1603.05766">1603.05766</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.2202%2F1544-6115.1585">10.2202/1544-6115.1585</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/21044043">21044043</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:10735784">10735784</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Statistical+Applications+in+Genetics+and+Molecular+Biology&rft.atitle=Permutation+p-values+should+never+be+zero%3A+calculating+exact+p-values+when+permutations+are+randomly+drawn&rft.volume=9&rft.issue=1&rft.pages=Article+39&rft.date=2010&rft_id=info%3Aarxiv%2F1603.05766&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A10735784%23id-name%3DS2CID&rft_id=info%3Apmid%2F21044043&rft_id=info%3Adoi%2F10.2202%2F1544-6115.1585&rft.aulast=Phipson&rft.aufirst=Belinda&rft.au=Smyth%2C+Gordon+K&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-4">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation journal cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20030505044125/http://tbf.coe.wayne.edu/jmasm/vol1_no2.pdf">"Invited Articles"</a> <span class="cs1-format">(PDF)</span>. <i><a href="/wiki/Journal_of_Modern_Applied_Statistical_Methods" title="Journal of Modern Applied Statistical Methods">Journal of Modern Applied Statistical Methods</a></i>. <b>1</b> (2): 202–522. Fall 2011. Archived from <a rel="nofollow" class="external text" href="http://tbf.coe.wayne.edu/jmasm/vol1_no2.pdf">the original</a> <span class="cs1-format">(PDF)</span> on May 5, 2003.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Modern+Applied+Statistical+Methods&rft.atitle=Invited+Articles&rft.ssn=fall&rft.volume=1&rft.issue=2&rft.pages=202-522&rft.date=2011&rft_id=http%3A%2F%2Ftbf.coe.wayne.edu%2Fjmasm%2Fvol1_no2.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-5">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCollingridge2012" class="citation journal cs1">Collingridge, Dave S. (11 September 2012). "A Primer on Quantitized Data Analysis and Permutation Testing". <i>Journal of Mixed Methods Research</i>. <b>7</b> (1): 81–97. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F1558689812454457">10.1177/1558689812454457</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:124618343">124618343</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Mixed+Methods+Research&rft.atitle=A+Primer+on+Quantitized+Data+Analysis+and+Permutation+Testing&rft.volume=7&rft.issue=1&rft.pages=81-97&rft.date=2012-09-11&rft_id=info%3Adoi%2F10.1177%2F1558689812454457&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A124618343%23id-name%3DS2CID&rft.aulast=Collingridge&rft.aufirst=Dave+S.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-6">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJanssen1997" class="citation journal cs1">Janssen, Arnold (1997). "Studentized Permutation Tests for Non-I.i.d. Hypotheses and the Generalized Behrens-Fisher Problem". <i>Statistics & Probability Letters</i>. <b>36</b> (1): 9–21. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fs0167-7152%2897%2900043-6">10.1016/s0167-7152(97)00043-6</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Statistics+%26+Probability+Letters&rft.atitle=Studentized+Permutation+Tests+for+Non-I.i.d.+Hypotheses+and+the+Generalized+Behrens-Fisher+Problem&rft.volume=36&rft.issue=1&rft.pages=9-21&rft.date=1997&rft_id=info%3Adoi%2F10.1016%2Fs0167-7152%2897%2900043-6&rft.aulast=Janssen&rft.aufirst=Arnold&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGood2005" class="citation book cs1">Good, Phillip I. (2005). <i>Resampling Methods: A Practical Guide to Data Analysis</i> (3rd ed.). Birkhäuser. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0817643867" title="Special:BookSources/978-0817643867"><bdi>978-0817643867</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Resampling+Methods%3A+A+Practical+Guide+to+Data+Analysis&rft.edition=3rd&rft.pub=Birkh%C3%A4user&rft.date=2005&rft.isbn=978-0817643867&rft.aulast=Good&rft.aufirst=Phillip+I.&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChungRomano2013" class="citation journal cs1">Chung, EY; Romano, JP (2013). <a rel="nofollow" class="external text" href="https://doi.org/10.1214%2F13-AOS1090">"Exact and asymptotically robust permutation tests"</a>. <i><a href="/wiki/The_Annals_of_Statistics" class="mw-redirect" title="The Annals of Statistics">The Annals of Statistics</a></i>. <b>41</b> (2): 487–507. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1304.5939">1304.5939</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1214%2F13-AOS1090">10.1214/13-AOS1090</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Annals+of+Statistics&rft.atitle=Exact+and+asymptotically+robust+permutation+tests&rft.volume=41&rft.issue=2&rft.pages=487-507&rft.date=2013&rft_id=info%3Aarxiv%2F1304.5939&rft_id=info%3Adoi%2F10.1214%2F13-AOS1090&rft.aulast=Chung&rft.aufirst=EY&rft.au=Romano%2C+JP&rft_id=https%3A%2F%2Fdoi.org%2F10.1214%252F13-AOS1090&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFEdenYates1933" class="citation journal cs1">Eden, T; Yates, F (1933). <a rel="nofollow" class="external text" href="https://www.cambridge.org/core/journals/journal-of-agricultural-science/article/abs/on-the-validity-of-fishers-z-test-when-applied-to-an-actual-example-of-nonnormal-data-with-five-textfigures/6232D2A79D698995B23E1A1AF4CEA8AB">"On the validity of Fisher's z test when applied to an actual example of non-normal data. (With five text-figures.)"</a>. <i>The Journal of Agricultural Science</i>. <b>23</b> (1): 6–17. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1017%2FS0021859600052862">10.1017/S0021859600052862</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:84802682">84802682</a><span class="reference-accessdate">. Retrieved <span class="nowrap">3 June</span> 2021</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Journal+of+Agricultural+Science&rft.atitle=On+the+validity+of+Fisher%27s+z+test+when+applied+to+an+actual+example+of+non-normal+data.+%28With+five+text-figures.%29&rft.volume=23&rft.issue=1&rft.pages=6-17&rft.date=1933&rft_id=info%3Adoi%2F10.1017%2FS0021859600052862&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A84802682%23id-name%3DS2CID&rft.aulast=Eden&rft.aufirst=T&rft.au=Yates%2C+F&rft_id=https%3A%2F%2Fwww.cambridge.org%2Fcore%2Fjournals%2Fjournal-of-agricultural-science%2Farticle%2Fabs%2Fon-the-validity-of-fishers-z-test-when-applied-to-an-actual-example-of-nonnormal-data-with-five-textfigures%2F6232D2A79D698995B23E1A1AF4CEA8AB&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-10"><span class="mw-cite-backlink"><b><a href="#cite_ref-10">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDwass1957" class="citation journal cs1">Dwass, Meyer (1957). <a rel="nofollow" class="external text" href="https://doi.org/10.1214%2Faoms%2F1177707045">"Modified Randomization Tests for Nonparametric Hypotheses"</a>. <i><a href="/wiki/Annals_of_Mathematical_Statistics" title="Annals of Mathematical Statistics">Annals of Mathematical Statistics</a></i>. <b>28</b> (1): 181–187. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1214%2Faoms%2F1177707045">10.1214/aoms/1177707045</a></span>. <a href="/wiki/JSTOR_(identifier)" class="mw-redirect" title="JSTOR (identifier)">JSTOR</a> <a rel="nofollow" class="external text" href="https://www.jstor.org/stable/2237031">2237031</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Annals+of+Mathematical+Statistics&rft.atitle=Modified+Randomization+Tests+for+Nonparametric+Hypotheses&rft.volume=28&rft.issue=1&rft.pages=181-187&rft.date=1957&rft_id=info%3Adoi%2F10.1214%2Faoms%2F1177707045&rft_id=https%3A%2F%2Fwww.jstor.org%2Fstable%2F2237031%23id-name%3DJSTOR&rft.aulast=Dwass&rft.aufirst=Meyer&rft_id=https%3A%2F%2Fdoi.org%2F10.1214%252Faoms%252F1177707045&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-11"><span class="mw-cite-backlink"><b><a href="#cite_ref-11">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFThomas_E._Nichols,_Andrew_P._Holmes2001" class="citation journal cs1"><a href="/wiki/Thomas_E._Nichols" title="Thomas E. Nichols">Thomas E. Nichols</a>, <a href="/w/index.php?title=Andrew_P._Holmes&action=edit&redlink=1" class="new" title="Andrew P. Holmes (page does not exist)">Andrew P. Holmes</a> (2001). <a rel="nofollow" class="external text" href="http://www.fil.ion.ucl.ac.uk/spm/doc/papers/NicholsHolmes.pdf">"Nonparametric Permutation Tests For Functional Neuroimaging: A Primer with Examples"</a> <span class="cs1-format">(PDF)</span>. <i><a href="/wiki/Human_Brain_Mapping_(journal)" title="Human Brain Mapping (journal)">Human Brain Mapping</a></i>. <b>15</b> (1): 1–25. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fhbm.1058">10.1002/hbm.1058</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/2027.42%2F35194">2027.42/35194</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6871862">6871862</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/11747097">11747097</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Human+Brain+Mapping&rft.atitle=Nonparametric+Permutation+Tests+For+Functional+Neuroimaging%3A+A+Primer+with+Examples&rft.volume=15&rft.issue=1&rft.pages=1-25&rft.date=2001&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6871862%23id-name%3DPMC&rft_id=info%3Apmid%2F11747097&rft_id=info%3Ahdl%2F2027.42%2F35194&rft_id=info%3Adoi%2F10.1002%2Fhbm.1058&rft.au=Thomas+E.+Nichols%2C+Andrew+P.+Holmes&rft_id=http%3A%2F%2Fwww.fil.ion.ucl.ac.uk%2Fspm%2Fdoc%2Fpapers%2FNicholsHolmes.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> <li id="cite_note-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-12">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGandy2009" class="citation journal cs1">Gandy, Axel (2009). "Sequential implementation of Monte Carlo tests with uniformly bounded resampling risk". <i><a href="/wiki/Journal_of_the_American_Statistical_Association" title="Journal of the American Statistical Association">Journal of the American Statistical Association</a></i>. <b>104</b> (488): 1504–1511. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/math/0612488">math/0612488</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1198%2Fjasa.2009.tm08368">10.1198/jasa.2009.tm08368</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15935787">15935787</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+American+Statistical+Association&rft.atitle=Sequential+implementation+of+Monte+Carlo+tests+with+uniformly+bounded+resampling+risk&rft.volume=104&rft.issue=488&rft.pages=1504-1511&rft.date=2009&rft_id=info%3Aarxiv%2Fmath%2F0612488&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15935787%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1198%2Fjasa.2009.tm08368&rft.aulast=Gandy&rft.aufirst=Axel&rfr_id=info%3Asid%2Fen.wikipedia.org%3APermutation+test" class="Z3988"></span></span> </li> </ol></div> <div class="navbox-styles"><style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="Statistics" style="padding:3px"><table class="nowraplinks hlist mw-collapsible uncollapsed navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Statistics" title="Template:Statistics"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Statistics" title="Template talk:Statistics"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Statistics" title="Special:EditPage/Template:Statistics"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Statistics" style="font-size:114%;margin:0 4em"><a href="/wiki/Statistics" title="Statistics">Statistics</a></div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><a href="/wiki/Outline_of_statistics" title="Outline of statistics">Outline</a></li> <li><a href="/wiki/List_of_statistics_articles" title="List of statistics articles">Index</a></li></ul> </div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Descriptive_statistics" style="font-size:114%;margin:0 4em"><a href="/wiki/Descriptive_statistics" title="Descriptive statistics">Descriptive statistics</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Continuous_probability_distribution" class="mw-redirect" title="Continuous probability distribution">Continuous data</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Central_tendency" title="Central tendency">Center</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Mean" title="Mean">Mean</a> <ul><li><a href="/wiki/Arithmetic_mean" title="Arithmetic mean">Arithmetic</a></li> <li><a href="/wiki/Arithmetic%E2%80%93geometric_mean" title="Arithmetic–geometric mean">Arithmetic-Geometric</a></li> <li><a href="/wiki/Contraharmonic_mean" title="Contraharmonic mean">Contraharmonic</a></li> <li><a href="/wiki/Cubic_mean" title="Cubic mean">Cubic</a></li> <li><a href="/wiki/Generalized_mean" title="Generalized mean">Generalized/power</a></li> <li><a href="/wiki/Geometric_mean" title="Geometric mean">Geometric</a></li> <li><a href="/wiki/Harmonic_mean" title="Harmonic mean">Harmonic</a></li> <li><a href="/wiki/Heronian_mean" title="Heronian mean">Heronian</a></li> <li><a href="/wiki/Heinz_mean" title="Heinz mean">Heinz</a></li> <li><a href="/wiki/Lehmer_mean" title="Lehmer mean">Lehmer</a></li></ul></li> <li><a href="/wiki/Median" title="Median">Median</a></li> <li><a href="/wiki/Mode_(statistics)" title="Mode (statistics)">Mode</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Statistical_dispersion" title="Statistical dispersion">Dispersion</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Average_absolute_deviation" title="Average absolute deviation">Average absolute deviation</a></li> <li><a href="/wiki/Coefficient_of_variation" title="Coefficient of variation">Coefficient of variation</a></li> <li><a href="/wiki/Interquartile_range" title="Interquartile range">Interquartile range</a></li> <li><a href="/wiki/Percentile" title="Percentile">Percentile</a></li> <li><a href="/wiki/Range_(statistics)" title="Range (statistics)">Range</a></li> <li><a href="/wiki/Standard_deviation" title="Standard deviation">Standard deviation</a></li> <li><a href="/wiki/Variance#Sample_variance" title="Variance">Variance</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Shape_of_the_distribution" class="mw-redirect" title="Shape of the distribution">Shape</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Central_limit_theorem" title="Central limit theorem">Central limit theorem</a></li> <li><a href="/wiki/Moment_(mathematics)" title="Moment (mathematics)">Moments</a> <ul><li><a href="/wiki/Kurtosis" title="Kurtosis">Kurtosis</a></li> <li><a href="/wiki/L-moment" title="L-moment">L-moments</a></li> <li><a href="/wiki/Skewness" title="Skewness">Skewness</a></li></ul></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Count_data" title="Count data">Count data</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Index_of_dispersion" title="Index of dispersion">Index of dispersion</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em">Summary tables</th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Contingency_table" title="Contingency table">Contingency table</a></li> <li><a href="/wiki/Frequency_distribution" class="mw-redirect" title="Frequency distribution">Frequency distribution</a></li> <li><a href="/wiki/Grouped_data" title="Grouped data">Grouped data</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">Dependence</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Partial_correlation" title="Partial correlation">Partial correlation</a></li> <li><a href="/wiki/Pearson_correlation_coefficient" title="Pearson correlation coefficient">Pearson product-moment correlation</a></li> <li><a href="/wiki/Rank_correlation" title="Rank correlation">Rank correlation</a> <ul><li><a href="/wiki/Kendall_rank_correlation_coefficient" title="Kendall rank correlation coefficient">Kendall's τ</a></li> <li><a href="/wiki/Spearman%27s_rank_correlation_coefficient" title="Spearman's rank correlation coefficient">Spearman's ρ</a></li></ul></li> <li><a href="/wiki/Scatter_plot" title="Scatter plot">Scatter plot</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Statistical_graphics" title="Statistical graphics">Graphics</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Bar_chart" title="Bar chart">Bar chart</a></li> <li><a href="/wiki/Biplot" title="Biplot">Biplot</a></li> <li><a href="/wiki/Box_plot" title="Box plot">Box plot</a></li> <li><a href="/wiki/Control_chart" title="Control chart">Control chart</a></li> <li><a href="/wiki/Correlogram" title="Correlogram">Correlogram</a></li> <li><a href="/wiki/Fan_chart_(statistics)" title="Fan chart (statistics)">Fan chart</a></li> <li><a href="/wiki/Forest_plot" title="Forest plot">Forest plot</a></li> <li><a href="/wiki/Histogram" title="Histogram">Histogram</a></li> <li><a href="/wiki/Pie_chart" title="Pie chart">Pie chart</a></li> <li><a href="/wiki/Q%E2%80%93Q_plot" title="Q–Q plot">Q–Q plot</a></li> <li><a href="/wiki/Radar_chart" title="Radar chart">Radar chart</a></li> <li><a href="/wiki/Run_chart" title="Run chart">Run chart</a></li> <li><a href="/wiki/Scatter_plot" title="Scatter plot">Scatter plot</a></li> <li><a href="/wiki/Stem-and-leaf_display" title="Stem-and-leaf display">Stem-and-leaf display</a></li> <li><a href="/wiki/Violin_plot" title="Violin plot">Violin plot</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Data_collection" style="font-size:114%;margin:0 4em"><a href="/wiki/Data_collection" title="Data collection">Data collection</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Design_of_experiments" title="Design of experiments">Study design</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Effect_size" title="Effect size">Effect size</a></li> <li><a href="/wiki/Missing_data" title="Missing data">Missing data</a></li> <li><a href="/wiki/Optimal_design" class="mw-redirect" title="Optimal design">Optimal design</a></li> <li><a href="/wiki/Statistical_population" title="Statistical population">Population</a></li> <li><a href="/wiki/Replication_(statistics)" title="Replication (statistics)">Replication</a></li> <li><a href="/wiki/Sample_size_determination" title="Sample size determination">Sample size determination</a></li> <li><a href="/wiki/Statistic" title="Statistic">Statistic</a></li> <li><a href="/wiki/Statistical_power" class="mw-redirect" title="Statistical power">Statistical power</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Survey_methodology" title="Survey methodology">Survey methodology</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Sampling_(statistics)" title="Sampling (statistics)">Sampling</a> <ul><li><a href="/wiki/Cluster_sampling" title="Cluster sampling">Cluster</a></li> <li><a href="/wiki/Stratified_sampling" title="Stratified sampling">Stratified</a></li></ul></li> <li><a href="/wiki/Opinion_poll" title="Opinion poll">Opinion poll</a></li> <li><a href="/wiki/Questionnaire" title="Questionnaire">Questionnaire</a></li> <li><a href="/wiki/Standard_error" title="Standard error">Standard error</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Experiment" title="Experiment">Controlled experiments</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Blocking_(statistics)" title="Blocking (statistics)">Blocking</a></li> <li><a href="/wiki/Factorial_experiment" title="Factorial experiment">Factorial experiment</a></li> <li><a href="/wiki/Interaction_(statistics)" title="Interaction (statistics)">Interaction</a></li> <li><a href="/wiki/Random_assignment" title="Random assignment">Random assignment</a></li> <li><a href="/wiki/Randomized_controlled_trial" title="Randomized controlled trial">Randomized controlled trial</a></li> <li><a href="/wiki/Randomized_experiment" title="Randomized experiment">Randomized experiment</a></li> <li><a href="/wiki/Scientific_control" title="Scientific control">Scientific control</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em">Adaptive designs</th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Adaptive_clinical_trial" class="mw-redirect" title="Adaptive clinical trial">Adaptive clinical trial</a></li> <li><a href="/wiki/Stochastic_approximation" title="Stochastic approximation">Stochastic approximation</a></li> <li><a href="/wiki/Up-and-Down_Designs" class="mw-redirect" title="Up-and-Down Designs">Up-and-down designs</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Observational_study" title="Observational study">Observational studies</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cohort_study" title="Cohort study">Cohort study</a></li> <li><a href="/wiki/Cross-sectional_study" title="Cross-sectional study">Cross-sectional study</a></li> <li><a href="/wiki/Natural_experiment" title="Natural experiment">Natural experiment</a></li> <li><a href="/wiki/Quasi-experiment" title="Quasi-experiment">Quasi-experiment</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible uncollapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Statistical_inference" style="font-size:114%;margin:0 4em"><a href="/wiki/Statistical_inference" title="Statistical inference">Statistical inference</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Statistical_theory" title="Statistical theory">Statistical theory</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Population_(statistics)" class="mw-redirect" title="Population (statistics)">Population</a></li> <li><a href="/wiki/Statistic" title="Statistic">Statistic</a></li> <li><a href="/wiki/Probability_distribution" title="Probability distribution">Probability distribution</a></li> <li><a href="/wiki/Sampling_distribution" title="Sampling distribution">Sampling distribution</a> <ul><li><a href="/wiki/Order_statistic" title="Order statistic">Order statistic</a></li></ul></li> <li><a href="/wiki/Empirical_distribution_function" title="Empirical distribution function">Empirical distribution</a> <ul><li><a href="/wiki/Density_estimation" title="Density estimation">Density estimation</a></li></ul></li> <li><a href="/wiki/Statistical_model" title="Statistical model">Statistical model</a> <ul><li><a href="/wiki/Model_specification" class="mw-redirect" title="Model specification">Model specification</a></li> <li><a href="/wiki/Lp_space" title="Lp space">L<sup><i>p</i></sup> space</a></li></ul></li> <li><a href="/wiki/Statistical_parameter" title="Statistical parameter">Parameter</a> <ul><li><a href="/wiki/Location_parameter" title="Location parameter">location</a></li> <li><a href="/wiki/Scale_parameter" title="Scale parameter">scale</a></li> <li><a href="/wiki/Shape_parameter" title="Shape parameter">shape</a></li></ul></li> <li><a href="/wiki/Parametric_statistics" title="Parametric statistics">Parametric family</a> <ul><li><a href="/wiki/Likelihood_function" title="Likelihood function">Likelihood</a> <a href="/wiki/Monotone_likelihood_ratio" title="Monotone likelihood ratio"><span style="font-size:85%;">(monotone)</span></a></li> <li><a href="/wiki/Location%E2%80%93scale_family" title="Location–scale family">Location–scale family</a></li> <li><a href="/wiki/Exponential_family" title="Exponential family">Exponential family</a></li></ul></li> <li><a href="/wiki/Completeness_(statistics)" title="Completeness (statistics)">Completeness</a></li> <li><a href="/wiki/Sufficient_statistic" title="Sufficient statistic">Sufficiency</a></li> <li><a href="/wiki/Plug-in_principle" class="mw-redirect" title="Plug-in principle">Statistical functional</a> <ul><li><a href="/wiki/Bootstrapping_(statistics)" title="Bootstrapping (statistics)">Bootstrap</a></li> <li><a href="/wiki/U-statistic" title="U-statistic">U</a></li> <li><a href="/wiki/V-statistic" title="V-statistic">V</a></li></ul></li> <li><a href="/wiki/Optimal_decision" title="Optimal decision">Optimal decision</a> <ul><li><a href="/wiki/Loss_function" title="Loss function">loss function</a></li></ul></li> <li><a href="/wiki/Efficiency_(statistics)" title="Efficiency (statistics)">Efficiency</a></li> <li><a href="/wiki/Statistical_distance" title="Statistical distance">Statistical distance</a> <ul><li><a href="/wiki/Divergence_(statistics)" title="Divergence (statistics)">divergence</a></li></ul></li> <li><a href="/wiki/Asymptotic_theory_(statistics)" title="Asymptotic theory (statistics)">Asymptotics</a></li> <li><a href="/wiki/Robust_statistics" title="Robust statistics">Robustness</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Frequentist_inference" title="Frequentist inference">Frequentist inference</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Point_estimation" title="Point estimation">Point estimation</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Estimating_equations" title="Estimating equations">Estimating equations</a> <ul><li><a href="/wiki/Maximum_likelihood" class="mw-redirect" title="Maximum likelihood">Maximum likelihood</a></li> <li><a href="/wiki/Method_of_moments_(statistics)" title="Method of moments (statistics)">Method of moments</a></li> <li><a href="/wiki/M-estimator" title="M-estimator">M-estimator</a></li> <li><a href="/wiki/Minimum_distance_estimation" class="mw-redirect" title="Minimum distance estimation">Minimum distance</a></li></ul></li> <li><a href="/wiki/Bias_of_an_estimator" title="Bias of an estimator">Unbiased estimators</a> <ul><li><a href="/wiki/Minimum-variance_unbiased_estimator" title="Minimum-variance unbiased estimator">Mean-unbiased minimum-variance</a> <ul><li><a href="/wiki/Rao%E2%80%93Blackwell_theorem" title="Rao–Blackwell theorem">Rao–Blackwellization</a></li> <li><a href="/wiki/Lehmann%E2%80%93Scheff%C3%A9_theorem" title="Lehmann–Scheffé theorem">Lehmann–Scheffé theorem</a></li></ul></li> <li><a href="/wiki/Median-unbiased_estimator" class="mw-redirect" title="Median-unbiased estimator">Median unbiased</a></li></ul></li> <li><a href="/wiki/Plug-in_principle" class="mw-redirect" title="Plug-in principle">Plug-in</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Interval_estimation" title="Interval estimation">Interval estimation</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Confidence_interval" title="Confidence interval">Confidence interval</a></li> <li><a href="/wiki/Pivotal_quantity" title="Pivotal quantity">Pivot</a></li> <li><a href="/wiki/Likelihood_interval" class="mw-redirect" title="Likelihood interval">Likelihood interval</a></li> <li><a href="/wiki/Prediction_interval" title="Prediction interval">Prediction interval</a></li> <li><a href="/wiki/Tolerance_interval" title="Tolerance interval">Tolerance interval</a></li> <li><a href="/wiki/Resampling_(statistics)" title="Resampling (statistics)">Resampling</a> <ul><li><a href="/wiki/Bootstrapping_(statistics)" title="Bootstrapping (statistics)">Bootstrap</a></li> <li><a href="/wiki/Jackknife_resampling" title="Jackknife resampling">Jackknife</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Statistical_hypothesis_testing" class="mw-redirect" title="Statistical hypothesis testing">Testing hypotheses</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/One-_and_two-tailed_tests" title="One- and two-tailed tests">1- & 2-tails</a></li> <li><a href="/wiki/Power_(statistics)" title="Power (statistics)">Power</a> <ul><li><a href="/wiki/Uniformly_most_powerful_test" title="Uniformly most powerful test">Uniformly most powerful test</a></li></ul></li> <li><a class="mw-selflink selflink">Permutation test</a> <ul><li><a href="/wiki/Randomization_test" class="mw-redirect" title="Randomization test">Randomization test</a></li></ul></li> <li><a href="/wiki/Multiple_comparisons" class="mw-redirect" title="Multiple comparisons">Multiple comparisons</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Parametric_statistics" title="Parametric statistics">Parametric tests</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Likelihood-ratio_test" title="Likelihood-ratio test">Likelihood-ratio</a></li> <li><a href="/wiki/Score_test" title="Score test">Score/Lagrange multiplier</a></li> <li><a href="/wiki/Wald_test" title="Wald test">Wald</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/List_of_statistical_tests" title="List of statistical tests">Specific tests</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><td colspan="2" class="navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Z-test" title="Z-test"><i>Z</i>-test <span style="font-size:85%;">(normal)</span></a></li> <li><a href="/wiki/Student%27s_t-test" title="Student's t-test">Student's <i>t</i>-test</a></li> <li><a href="/wiki/F-test" title="F-test"><i>F</i>-test</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Goodness_of_fit" title="Goodness of fit">Goodness of fit</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Chi-squared_test" title="Chi-squared test">Chi-squared</a></li> <li><a href="/wiki/G-test" title="G-test"><i>G</i>-test</a></li> <li><a href="/wiki/Kolmogorov%E2%80%93Smirnov_test" title="Kolmogorov–Smirnov test">Kolmogorov–Smirnov</a></li> <li><a href="/wiki/Anderson%E2%80%93Darling_test" title="Anderson–Darling test">Anderson–Darling</a></li> <li><a href="/wiki/Lilliefors_test" title="Lilliefors test">Lilliefors</a></li> <li><a href="/wiki/Jarque%E2%80%93Bera_test" title="Jarque–Bera test">Jarque–Bera</a></li> <li><a href="/wiki/Shapiro%E2%80%93Wilk_test" title="Shapiro–Wilk test">Normality <span style="font-size:85%;">(Shapiro–Wilk)</span></a></li> <li><a href="/wiki/Likelihood-ratio_test" title="Likelihood-ratio test">Likelihood-ratio test</a></li> <li><a href="/wiki/Model_selection" title="Model selection">Model selection</a> <ul><li><a href="/wiki/Cross-validation_(statistics)" title="Cross-validation (statistics)">Cross validation</a></li> <li><a href="/wiki/Akaike_information_criterion" title="Akaike information criterion">AIC</a></li> <li><a href="/wiki/Bayesian_information_criterion" title="Bayesian information criterion">BIC</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Rank_statistics" class="mw-redirect" title="Rank statistics">Rank statistics</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Sign_test" title="Sign test">Sign</a> <ul><li><a href="/wiki/Sample_median" class="mw-redirect" title="Sample median">Sample median</a></li></ul></li> <li><a href="/wiki/Wilcoxon_signed-rank_test" title="Wilcoxon signed-rank test">Signed rank <span style="font-size:85%;">(Wilcoxon)</span></a> <ul><li><a href="/wiki/Hodges%E2%80%93Lehmann_estimator" title="Hodges–Lehmann estimator">Hodges–Lehmann estimator</a></li></ul></li> <li><a href="/wiki/Mann%E2%80%93Whitney_U_test" title="Mann–Whitney U test">Rank sum <span style="font-size:85%;">(Mann–Whitney)</span></a></li> <li><a href="/wiki/Nonparametric_statistics" title="Nonparametric statistics">Nonparametric</a> <a href="/wiki/Analysis_of_variance" title="Analysis of variance">anova</a> <ul><li><a href="/wiki/Kruskal%E2%80%93Wallis_test" title="Kruskal–Wallis test">1-way <span style="font-size:85%;">(Kruskal–Wallis)</span></a></li> <li><a href="/wiki/Friedman_test" title="Friedman test">2-way <span style="font-size:85%;">(Friedman)</span></a></li> <li><a href="/wiki/Jonckheere%27s_trend_test" title="Jonckheere's trend test">Ordered alternative <span style="font-size:85%;">(Jonckheere–Terpstra)</span></a></li></ul></li> <li><a href="/wiki/Van_der_Waerden_test" title="Van der Waerden test">Van der Waerden test</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Bayesian_inference" title="Bayesian inference">Bayesian inference</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Bayesian_probability" title="Bayesian probability">Bayesian probability</a> <ul><li><a href="/wiki/Prior_probability" title="Prior probability">prior</a></li> <li><a href="/wiki/Posterior_probability" title="Posterior probability">posterior</a></li></ul></li> <li><a href="/wiki/Credible_interval" title="Credible interval">Credible interval</a></li> <li><a href="/wiki/Bayes_factor" title="Bayes factor">Bayes factor</a></li> <li><a href="/wiki/Bayes_estimator" title="Bayes estimator">Bayesian estimator</a> <ul><li><a href="/wiki/Maximum_a_posteriori_estimation" title="Maximum a posteriori estimation">Maximum posterior estimator</a></li></ul></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="CorrelationRegression_analysis" style="font-size:114%;margin:0 4em"><div class="hlist"><ul><li><a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">Correlation</a></li><li><a href="/wiki/Regression_analysis" title="Regression analysis">Regression analysis</a></li></ul></div></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">Correlation</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Pearson_product-moment_correlation_coefficient" class="mw-redirect" title="Pearson product-moment correlation coefficient">Pearson product-moment</a></li> <li><a href="/wiki/Partial_correlation" title="Partial correlation">Partial correlation</a></li> <li><a href="/wiki/Confounding" title="Confounding">Confounding variable</a></li> <li><a href="/wiki/Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Regression_analysis" title="Regression analysis">Regression analysis</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Errors_and_residuals" title="Errors and residuals">Errors and residuals</a></li> <li><a href="/wiki/Regression_validation" title="Regression validation">Regression validation</a></li> <li><a href="/wiki/Mixed_model" title="Mixed model">Mixed effects models</a></li> <li><a href="/wiki/Simultaneous_equations_model" title="Simultaneous equations model">Simultaneous equations models</a></li> <li><a href="/wiki/Multivariate_adaptive_regression_splines" class="mw-redirect" title="Multivariate adaptive regression splines">Multivariate adaptive regression splines (MARS)</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Linear_regression" title="Linear regression">Linear regression</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Simple_linear_regression" title="Simple linear regression">Simple linear regression</a></li> <li><a href="/wiki/Ordinary_least_squares" title="Ordinary least squares">Ordinary least squares</a></li> <li><a href="/wiki/General_linear_model" title="General linear model">General linear model</a></li> <li><a href="/wiki/Bayesian_linear_regression" title="Bayesian linear regression">Bayesian regression</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em">Non-standard predictors</th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Nonlinear_regression" title="Nonlinear regression">Nonlinear regression</a></li> <li><a href="/wiki/Nonparametric_regression" title="Nonparametric regression">Nonparametric</a></li> <li><a href="/wiki/Semiparametric_regression" title="Semiparametric regression">Semiparametric</a></li> <li><a href="/wiki/Isotonic_regression" title="Isotonic regression">Isotonic</a></li> <li><a href="/wiki/Robust_regression" title="Robust regression">Robust</a></li> <li><a href="/wiki/Heteroscedasticity" class="mw-redirect" title="Heteroscedasticity">Heteroscedasticity</a></li> <li><a href="/wiki/Homoscedasticity" class="mw-redirect" title="Homoscedasticity">Homoscedasticity</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Generalized_linear_model" title="Generalized linear model">Generalized linear model</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Exponential_family" title="Exponential family">Exponential families</a></li> <li><a href="/wiki/Logistic_regression" title="Logistic regression">Logistic <span style="font-size:85%;">(Bernoulli)</span></a> / <a href="/wiki/Binomial_regression" title="Binomial regression">Binomial</a> / <a href="/wiki/Poisson_regression" title="Poisson regression">Poisson regressions</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Partition_of_sums_of_squares" title="Partition of sums of squares">Partition of variance</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Analysis_of_variance" title="Analysis of variance">Analysis of variance (ANOVA, anova)</a></li> <li><a href="/wiki/Analysis_of_covariance" title="Analysis of covariance">Analysis of covariance</a></li> <li><a href="/wiki/Multivariate_analysis_of_variance" title="Multivariate analysis of variance">Multivariate ANOVA</a></li> <li><a href="/wiki/Degrees_of_freedom_(statistics)" title="Degrees of freedom (statistics)">Degrees of freedom</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Categorical_/_Multivariate_/_Time-series_/_Survival_analysis" style="font-size:114%;margin:0 4em"><a href="/wiki/Categorical_variable" title="Categorical variable">Categorical</a> / <a href="/wiki/Multivariate_statistics" title="Multivariate statistics">Multivariate</a> / <a href="/wiki/Time_series" title="Time series">Time-series</a> / <a href="/wiki/Survival_analysis" title="Survival analysis">Survival analysis</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Categorical_variable" title="Categorical variable">Categorical</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cohen%27s_kappa" title="Cohen's kappa">Cohen's kappa</a></li> <li><a href="/wiki/Contingency_table" title="Contingency table">Contingency table</a></li> <li><a href="/wiki/Graphical_model" title="Graphical model">Graphical model</a></li> <li><a href="/wiki/Poisson_regression" title="Poisson regression">Log-linear model</a></li> <li><a href="/wiki/McNemar%27s_test" title="McNemar's test">McNemar's test</a></li> <li><a href="/wiki/Cochran%E2%80%93Mantel%E2%80%93Haenszel_statistics" title="Cochran–Mantel–Haenszel statistics">Cochran–Mantel–Haenszel statistics</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Multivariate_statistics" title="Multivariate statistics">Multivariate</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/General_linear_model" title="General linear model">Regression</a></li> <li><a href="/wiki/Multivariate_analysis_of_variance" title="Multivariate analysis of variance">Manova</a></li> <li><a href="/wiki/Principal_component_analysis" title="Principal component analysis">Principal components</a></li> <li><a href="/wiki/Canonical_correlation" title="Canonical correlation">Canonical correlation</a></li> <li><a href="/wiki/Linear_discriminant_analysis" title="Linear discriminant analysis">Discriminant analysis</a></li> <li><a href="/wiki/Cluster_analysis" title="Cluster analysis">Cluster analysis</a></li> <li><a href="/wiki/Statistical_classification" title="Statistical classification">Classification</a></li> <li><a href="/wiki/Structural_equation_modeling" title="Structural equation modeling">Structural equation model</a> <ul><li><a href="/wiki/Factor_analysis" title="Factor analysis">Factor analysis</a></li></ul></li> <li><a href="/wiki/Multivariate_distribution" class="mw-redirect" title="Multivariate distribution">Multivariate distributions</a> <ul><li><a href="/wiki/Elliptical_distribution" title="Elliptical distribution">Elliptical distributions</a> <ul><li><a href="/wiki/Multivariate_normal_distribution" title="Multivariate normal distribution">Normal</a></li></ul></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Time_series" title="Time series">Time-series</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;">General</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Decomposition_of_time_series" title="Decomposition of time series">Decomposition</a></li> <li><a href="/wiki/Trend_estimation" class="mw-redirect" title="Trend estimation">Trend</a></li> <li><a href="/wiki/Stationary_process" title="Stationary process">Stationarity</a></li> <li><a href="/wiki/Seasonal_adjustment" title="Seasonal adjustment">Seasonal adjustment</a></li> <li><a href="/wiki/Exponential_smoothing" title="Exponential smoothing">Exponential smoothing</a></li> <li><a href="/wiki/Cointegration" title="Cointegration">Cointegration</a></li> <li><a href="/wiki/Structural_break" title="Structural break">Structural break</a></li> <li><a href="/wiki/Granger_causality" title="Granger causality">Granger causality</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;">Specific tests</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Dickey%E2%80%93Fuller_test" title="Dickey–Fuller test">Dickey–Fuller</a></li> <li><a href="/wiki/Johansen_test" title="Johansen test">Johansen</a></li> <li><a href="/wiki/Ljung%E2%80%93Box_test" title="Ljung–Box test">Q-statistic <span style="font-size:85%;">(Ljung–Box)</span></a></li> <li><a href="/wiki/Durbin%E2%80%93Watson_statistic" title="Durbin–Watson statistic">Durbin–Watson</a></li> <li><a href="/wiki/Breusch%E2%80%93Godfrey_test" title="Breusch–Godfrey test">Breusch–Godfrey</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Time_domain" title="Time domain">Time domain</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Autocorrelation" title="Autocorrelation">Autocorrelation (ACF)</a> <ul><li><a href="/wiki/Partial_autocorrelation_function" title="Partial autocorrelation function">partial (PACF)</a></li></ul></li> <li><a href="/wiki/Cross-correlation" title="Cross-correlation">Cross-correlation (XCF)</a></li> <li><a href="/wiki/Autoregressive%E2%80%93moving-average_model" class="mw-redirect" title="Autoregressive–moving-average model">ARMA model</a></li> <li><a href="/wiki/Box%E2%80%93Jenkins_method" title="Box–Jenkins method">ARIMA model <span style="font-size:85%;">(Box–Jenkins)</span></a></li> <li><a href="/wiki/Autoregressive_conditional_heteroskedasticity" title="Autoregressive conditional heteroskedasticity">Autoregressive conditional heteroskedasticity (ARCH)</a></li> <li><a href="/wiki/Vector_autoregression" title="Vector autoregression">Vector autoregression (VAR)</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Frequency_domain" title="Frequency domain">Frequency domain</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Spectral_density_estimation" title="Spectral density estimation">Spectral density estimation</a></li> <li><a href="/wiki/Fourier_analysis" title="Fourier analysis">Fourier analysis</a></li> <li><a href="/wiki/Least-squares_spectral_analysis" title="Least-squares spectral analysis">Least-squares spectral analysis</a></li> <li><a href="/wiki/Wavelet" title="Wavelet">Wavelet</a></li> <li><a href="/wiki/Whittle_likelihood" title="Whittle likelihood">Whittle likelihood</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Survival_analysis" title="Survival analysis">Survival</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Survival_function" title="Survival function">Survival function</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Kaplan%E2%80%93Meier_estimator" title="Kaplan–Meier estimator">Kaplan–Meier estimator (product limit)</a></li> <li><a href="/wiki/Proportional_hazards_model" title="Proportional hazards model">Proportional hazards models</a></li> <li><a href="/wiki/Accelerated_failure_time_model" title="Accelerated failure time model">Accelerated failure time (AFT) model</a></li> <li><a href="/wiki/First-hitting-time_model" title="First-hitting-time model">First hitting time</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Failure_rate" title="Failure rate">Hazard function</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Nelson%E2%80%93Aalen_estimator" title="Nelson–Aalen estimator">Nelson–Aalen estimator</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;">Test</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Log-rank_test" class="mw-redirect" title="Log-rank test">Log-rank test</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Applications" style="font-size:114%;margin:0 4em"><a href="/wiki/List_of_fields_of_application_of_statistics" title="List of fields of application of statistics">Applications</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Biostatistics" title="Biostatistics">Biostatistics</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Bioinformatics" title="Bioinformatics">Bioinformatics</a></li> <li><a href="/wiki/Clinical_trial" title="Clinical trial">Clinical trials</a> / <a href="/wiki/Clinical_study_design" title="Clinical study design">studies</a></li> <li><a href="/wiki/Epidemiology" title="Epidemiology">Epidemiology</a></li> <li><a href="/wiki/Medical_statistics" title="Medical statistics">Medical statistics</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Engineering_statistics" title="Engineering statistics">Engineering statistics</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Chemometrics" title="Chemometrics">Chemometrics</a></li> <li><a href="/wiki/Methods_engineering" title="Methods engineering">Methods engineering</a></li> <li><a href="/wiki/Probabilistic_design" title="Probabilistic design">Probabilistic design</a></li> <li><a href="/wiki/Statistical_process_control" title="Statistical process control">Process</a> / <a href="/wiki/Quality_control" title="Quality control">quality control</a></li> <li><a href="/wiki/Reliability_engineering" title="Reliability engineering">Reliability</a></li> <li><a href="/wiki/System_identification" title="System identification">System identification</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Social_statistics" title="Social statistics">Social statistics</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Actuarial_science" title="Actuarial science">Actuarial science</a></li> <li><a href="/wiki/Census" title="Census">Census</a></li> <li><a href="/wiki/Crime_statistics" title="Crime statistics">Crime statistics</a></li> <li><a href="/wiki/Demographic_statistics" title="Demographic statistics">Demography</a></li> <li><a href="/wiki/Econometrics" title="Econometrics">Econometrics</a></li> <li><a href="/wiki/Jurimetrics" title="Jurimetrics">Jurimetrics</a></li> <li><a href="/wiki/National_accounts" title="National accounts">National accounts</a></li> <li><a href="/wiki/Official_statistics" title="Official statistics">Official statistics</a></li> <li><a href="/wiki/Population_statistics" class="mw-redirect" title="Population statistics">Population statistics</a></li> <li><a href="/wiki/Psychometrics" title="Psychometrics">Psychometrics</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Spatial_analysis" title="Spatial analysis">Spatial statistics</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cartography" title="Cartography">Cartography</a></li> <li><a href="/wiki/Environmental_statistics" title="Environmental statistics">Environmental statistics</a></li> <li><a href="/wiki/Geographic_information_system" title="Geographic information system">Geographic information system</a></li> <li><a href="/wiki/Geostatistics" title="Geostatistics">Geostatistics</a></li> <li><a href="/wiki/Kriging" title="Kriging">Kriging</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><span class="noviewer" typeof="mw:File"><span title="Category"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/16px-Symbol_category_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/23px-Symbol_category_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/31px-Symbol_category_class.svg.png 2x" data-file-width="180" data-file-height="185" /></span></span><b><a href="/wiki/Category:Statistics" title="Category:Statistics">Category</a></b></li> <li><b><span class="nowrap"><span class="noviewer" typeof="mw:File"><a href="/wiki/File:Nuvola_apps_edu_mathematics_blue-p.svg" class="mw-file-description"><img alt="icon" src="//upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Nuvola_apps_edu_mathematics_blue-p.svg/28px-Nuvola_apps_edu_mathematics_blue-p.svg.png" decoding="async" width="28" height="28" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Nuvola_apps_edu_mathematics_blue-p.svg/42px-Nuvola_apps_edu_mathematics_blue-p.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Nuvola_apps_edu_mathematics_blue-p.svg/56px-Nuvola_apps_edu_mathematics_blue-p.svg.png 2x" data-file-width="128" data-file-height="128" /></a></span> </span><a href="/wiki/Portal:Mathematics" title="Portal:Mathematics">Mathematics portal</a></b></li> <li><span class="noviewer" typeof="mw:File"><span title="Commons page"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/12px-Commons-logo.svg.png" decoding="async" width="12" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/18px-Commons-logo.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/24px-Commons-logo.svg.png 2x" data-file-width="1024" data-file-height="1376" /></span></span><b><a href="https://commons.wikimedia.org/wiki/Category:Statistics" class="extiw" title="commons:Category:Statistics">Commons</a></b></li> <li><span class="noviewer" typeof="mw:File"><span title="WikiProject"><img alt="" src="//upload.wikimedia.org/wikipedia/commons/thumb/3/37/People_icon.svg/16px-People_icon.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/37/People_icon.svg/24px-People_icon.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/37/People_icon.svg/32px-People_icon.svg.png 2x" data-file-width="100" data-file-height="100" /></span></span> <b><a href="/wiki/Wikipedia:WikiProject_Statistics" title="Wikipedia:WikiProject Statistics">WikiProject</a></b></li></ul> </div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Permutation_test&oldid=1255317817">https://en.wikipedia.org/w/index.php?title=Permutation_test&oldid=1255317817</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Statistical_tests" title="Category:Statistical tests">Statistical tests</a></li><li><a href="/wiki/Category:Nonparametric_statistics" title="Category:Nonparametric statistics">Nonparametric statistics</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_matches_Wikidata" title="Category:Short description matches Wikidata">Short description matches Wikidata</a></li><li><a href="/wiki/Category:All_articles_with_unsourced_statements" title="Category:All articles with unsourced statements">All articles with unsourced statements</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_January_2020" title="Category:Articles with unsourced statements from January 2020">Articles with unsourced statements from January 2020</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 4 November 2024, at 08:38<span class="anonymous-show"> (UTC)</span>.</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Permutation_test&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-6f5tb","wgBackendResponseTime":152,"wgPageParseReport":{"limitreport":{"cputime":"0.524","walltime":"0.761","ppvisitednodes":{"value":3015,"limit":1000000},"postexpandincludesize":{"value":187431,"limit":2097152},"templateargumentsize":{"value":1955,"limit":2097152},"expansiondepth":{"value":14,"limit":100},"expensivefunctioncount":{"value":3,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":76699,"limit":5000000},"entityaccesscount":{"value":0,"limit":400},"timingprofile":["100.00% 540.757 1 -total"," 30.32% 163.937 16 Template:Cite_journal"," 29.28% 158.312 1 Template:Statistics"," 28.70% 155.188 1 Template:Navbox_with_collapsible_groups"," 17.89% 96.734 1 Template:Short_description"," 11.69% 63.234 2 Template:Pagetype"," 8.92% 48.214 1 Template:Hlist"," 7.97% 43.124 4 Template:ISBN"," 7.56% 40.896 1 Template:Citation_needed"," 6.61% 35.758 1 Template:Fix"]},"scribunto":{"limitreport-timeusage":{"value":"0.289","limit":"10.000"},"limitreport-memusage":{"value":5950533,"limit":52428800},"limitreport-logs":"table#1 {\n}\n"},"cachereport":{"origin":"mw-web.eqiad.canary-687bc74f68-7w47t","timestamp":"20241122141454","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Permutation test","url":"https:\/\/en.wikipedia.org\/wiki\/Permutation_test","sameAs":"http:\/\/www.wikidata.org\/entity\/Q108839445","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q108839445","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2005-08-16T18:45:23Z","dateModified":"2024-11-04T08:38:12Z","headline":"exact statistical hypothesis test"}</script> </body> </html>