CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Cohen's kappa - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"6640b19a-a28e-48e7-93e2-6e13ab3c72cd","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Cohen's_kappa","wgTitle":"Cohen's kappa","wgCurRevisionId":1245316837,"wgRevisionId":1245316837,"wgArticleId":1701650,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Wikipedia articles needing page number citations from April 2012","Articles with short description","Short description matches Wikidata","All articles with unsourced statements","Articles with unsourced statements from April 2012","Categorical variable interactions","Nonparametric statistics","Inter-rater reliability","Summary statistics for contingency tables"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Cohen's_kappa" ,"wgRelevantArticleId":1701650,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":30000,"wgRelatedArticlesCompat":[],"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q1107106","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform" ,"platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.cite.styles":"ready","ext.math.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","mediawiki.page.media","ext.scribunto.logs","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips" ,"ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession","wikibase.sidebar.tracking"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Cohen's kappa - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Cohen%27s_kappa"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Cohen%27s_kappa&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Cohen%27s_kappa"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Cohen_s_kappa rootpage-Cohen_s_kappa skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia"><span>Contents</span></a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events"><span>Current events</span></a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x"><span>Random article</span></a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works"><span>About Wikipedia</span></a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia"><span>Contact us</span></a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia"><span>Help</span></a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia"><span>Learn to edit</span></a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors"><span>Community portal</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia"><span>Upload file</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Cohen%27s+kappa" title="You are encouraged to create an account and log in; however, it is not mandatory" class=""><span>Create account</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Cohen%27s+kappa" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Cohen%27s+kappa" title="You are encouraged to create an account and log in; however, it is not mandatory"><span class="vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd"></span> <span>Create account</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Cohen%27s+kappa" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing"><span>learn more</span></a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y"><span>Contributions</span></a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n"><span>Talk</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-History" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#History"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>History</span> </div> </a> <ul id="toc-History-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Definition" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Definition"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Definition</span> </div> </a> <button aria-controls="toc-Definition-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Definition subsection</span> </button> <ul id="toc-Definition-sublist" class="vector-toc-list"> <li id="toc-Binary_classification_confusion_matrix" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Binary_classification_confusion_matrix"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1</span> <span>Binary classification confusion matrix</span> </div> </a> <ul id="toc-Binary_classification_confusion_matrix-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Examples" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Examples"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Examples</span> </div> </a> <button aria-controls="toc-Examples-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Examples subsection</span> </button> <ul id="toc-Examples-sublist" class="vector-toc-list"> <li id="toc-Simple_example" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Simple_example"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Simple example</span> </div> </a> <ul id="toc-Simple_example-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Same_percentages_but_different_numbers" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Same_percentages_but_different_numbers"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.2</span> <span>Same percentages but different numbers</span> </div> </a> <ul id="toc-Same_percentages_but_different_numbers-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Properties" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Properties"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Properties</span> </div> </a> <button aria-controls="toc-Properties-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Properties subsection</span> </button> <ul id="toc-Properties-sublist" class="vector-toc-list"> <li id="toc-Hypothesis_testing_and_confidence_interval" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Hypothesis_testing_and_confidence_interval"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.1</span> <span>Hypothesis testing and confidence interval</span> </div> </a> <ul id="toc-Hypothesis_testing_and_confidence_interval-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Interpreting_magnitude" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Interpreting_magnitude"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.2</span> <span>Interpreting magnitude</span> </div> </a> <ul id="toc-Interpreting_magnitude-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Kappa_maximum" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Kappa_maximum"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.3</span> <span>Kappa maximum</span> </div> </a> <ul id="toc-Kappa_maximum-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Limitations" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Limitations"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4</span> <span>Limitations</span> </div> </a> <ul id="toc-Limitations-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Related_statistics" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Related_statistics"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Related statistics</span> </div> </a> <button aria-controls="toc-Related_statistics-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Related statistics subsection</span> </button> <ul id="toc-Related_statistics-sublist" class="vector-toc-list"> <li id="toc-Scott's_Pi" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Scott's_Pi"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.1</span> <span>Scott's Pi</span> </div> </a> <ul id="toc-Scott's_Pi-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Fleiss'_kappa" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Fleiss'_kappa"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.2</span> <span>Fleiss' kappa</span> </div> </a> <ul id="toc-Fleiss'_kappa-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Weighted_kappa" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Weighted_kappa"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.3</span> <span>Weighted kappa</span> </div> </a> <ul id="toc-Weighted_kappa-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>See also</span> </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Further_reading" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Further_reading"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>Further reading</span> </div> </a> <ul id="toc-Further_reading-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-External_links" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#External_links"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>External links</span> </div> </a> <ul id="toc-External_links-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Cohen's kappa</span></h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 17 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-17" aria-hidden="true" ><span class="vector-icon mw-ui-icon-language-progressive mw-ui-icon-wikimedia-language-progressive"></span> <span class="vector-dropdown-label-text">17 languages</span> </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D9%85%D8%B9%D8%A7%D9%85%D9%84_%D9%83%D9%88%D9%87%D9%8A%D9%86_%D9%83%D8%A7%D8%A8%D8%A7" title="معامل كوهين كابا – Arabic" lang="ar" hreflang="ar" data-title="معامل كوهين كابا" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target"><span>العربية</span></a></li><li class="interlanguage-link interwiki-ca mw-list-item"><a href="https://ca.wikipedia.org/wiki/Coeficient_kappa_de_Cohen" title="Coeficient kappa de Cohen – Catalan" lang="ca" hreflang="ca" data-title="Coeficient kappa de Cohen" data-language-autonym="Català" data-language-local-name="Catalan" class="interlanguage-link-target"><span>Català</span></a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Cohens_Kappa" title="Cohens Kappa – German" lang="de" hreflang="de" data-title="Cohens Kappa" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target"><span>Deutsch</span></a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Coeficiente_kappa_de_Cohen" title="Coeficiente kappa de Cohen – Spanish" lang="es" hreflang="es" data-title="Coeficiente kappa de Cohen" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target"><span>Español</span></a></li><li class="interlanguage-link interwiki-eu mw-list-item"><a href="https://eu.wikipedia.org/wiki/Cohenen_kappa" title="Cohenen kappa – Basque" lang="eu" hreflang="eu" data-title="Cohenen kappa" data-language-autonym="Euskara" data-language-local-name="Basque" class="interlanguage-link-target"><span>Euskara</span></a></li><li class="interlanguage-link interwiki-fa mw-list-item"><a href="https://fa.wikipedia.org/wiki/%DA%A9%D8%A7%D9%BE%D8%A7%DB%8C_%DA%A9%D9%88%D9%87%D9%86" title="کاپای کوهن – Persian" lang="fa" hreflang="fa" data-title="کاپای کوهن" data-language-autonym="فارسی" data-language-local-name="Persian" class="interlanguage-link-target"><span>فارسی</span></a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/Kappa_de_Cohen" title="Kappa de Cohen – French" lang="fr" hreflang="fr" data-title="Kappa de Cohen" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target"><span>Français</span></a></li><li class="interlanguage-link interwiki-ko mw-list-item"><a href="https://ko.wikipedia.org/wiki/%EC%B9%B4%ED%8C%8C_%EC%83%81%EA%B4%80%EA%B3%84%EC%88%98" title="카파 상관계수 – Korean" lang="ko" hreflang="ko" data-title="카파 상관계수" data-language-autonym="한국어" data-language-local-name="Korean" class="interlanguage-link-target"><span>한국어</span></a></li><li class="interlanguage-link interwiki-it mw-list-item"><a href="https://it.wikipedia.org/wiki/Kappa_di_Cohen" title="Kappa di Cohen – Italian" lang="it" hreflang="it" data-title="Kappa di Cohen" data-language-autonym="Italiano" data-language-local-name="Italian" class="interlanguage-link-target"><span>Italiano</span></a></li><li class="interlanguage-link interwiki-hu mw-list-item"><a href="https://hu.wikipedia.org/wiki/Cohen-f%C3%A9le_kappa" title="Cohen-féle kappa – Hungarian" lang="hu" hreflang="hu" data-title="Cohen-féle kappa" data-language-autonym="Magyar" data-language-local-name="Hungarian" class="interlanguage-link-target"><span>Magyar</span></a></li><li class="interlanguage-link interwiki-pl mw-list-item"><a href="https://pl.wikipedia.org/wiki/Kappa_Cohena" title="Kappa Cohena – Polish" lang="pl" hreflang="pl" data-title="Kappa Cohena" data-language-autonym="Polski" data-language-local-name="Polish" class="interlanguage-link-target"><span>Polski</span></a></li><li class="interlanguage-link interwiki-pt mw-list-item"><a href="https://pt.wikipedia.org/wiki/Kappa_de_Cohen" title="Kappa de Cohen – Portuguese" lang="pt" hreflang="pt" data-title="Kappa de Cohen" data-language-autonym="Português" data-language-local-name="Portuguese" class="interlanguage-link-target"><span>Português</span></a></li><li class="interlanguage-link interwiki-si badge-Q17437798 badge-goodarticle mw-list-item" title="good article badge"><a href="https://si.wikipedia.org/wiki/%E0%B7%86%E0%B7%8A%E0%B6%BD%E0%B6%BA%E0%B7%92%E0%B7%83%E0%B7%8A%E0%B6%9C%E0%B7%9A_%E0%B6%9A%E0%B7%90%E0%B6%B4%E0%B7%8F_%E0%B7%80%E0%B7%92%E0%B7%81%E0%B7%8A%E0%B6%BD%E0%B7%9A%E0%B7%82%E0%B6%AB%E0%B6%BA" title="ෆ්ලයිස්ගේ කැපා විශ්ලේෂණය – Sinhala" lang="si" hreflang="si" data-title="ෆ්ලයිස්ගේ කැපා විශ්ලේෂණය" data-language-autonym="සිංහල" data-language-local-name="Sinhala" class="interlanguage-link-target"><span>සිංහල</span></a></li><li class="interlanguage-link interwiki-fi mw-list-item"><a href="https://fi.wikipedia.org/wiki/Cohenin_kappa" title="Cohenin kappa – Finnish" lang="fi" hreflang="fi" data-title="Cohenin kappa" data-language-autonym="Suomi" data-language-local-name="Finnish" class="interlanguage-link-target"><span>Suomi</span></a></li><li class="interlanguage-link interwiki-th mw-list-item"><a href="https://th.wikipedia.org/wiki/%E0%B8%AA%E0%B8%B1%E0%B8%A1%E0%B8%9B%E0%B8%A3%E0%B8%B0%E0%B8%AA%E0%B8%B4%E0%B8%97%E0%B8%98%E0%B8%B4%E0%B9%8C%E0%B9%81%E0%B8%84%E0%B8%9B%E0%B8%9B%E0%B8%B2%E0%B8%82%E0%B8%AD%E0%B8%87%E0%B9%82%E0%B8%84%E0%B9%80%E0%B8%AE%E0%B8%99" title="สัมประสิทธิ์แคปปาของโคเฮน – Thai" lang="th" hreflang="th" data-title="สัมประสิทธิ์แคปปาของโคเฮน" data-language-autonym="ไทย" data-language-local-name="Thai" class="interlanguage-link-target"><span>ไทย</span></a></li><li class="interlanguage-link interwiki-tr mw-list-item"><a href="https://tr.wikipedia.org/wiki/Cohen%27in_kappa_katsay%C4%B1s%C4%B1" title="Cohen'in kappa katsayısı – Turkish" lang="tr" hreflang="tr" data-title="Cohen'in kappa katsayısı" data-language-autonym="Türkçe" data-language-local-name="Turkish" class="interlanguage-link-target"><span>Türkçe</span></a></li><li class="interlanguage-link interwiki-zh-yue mw-list-item"><a href="https://zh-yue.wikipedia.org/wiki/%E9%AB%98%E6%B0%8F_kappa" title="高氏 kappa – Cantonese" lang="yue" hreflang="yue" data-title="高氏 kappa" data-language-autonym="粵語" data-language-local-name="Cantonese" class="interlanguage-link-target"><span>粵語</span></a></li> </ul> <div class="after-portlet after-portlet-lang"><span class="wb-langlinks-edit wb-langlinks-link"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q1107106#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></span></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Cohen%27s_kappa" title="View the content page [c]" accesskey="c"><span>Article</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Cohen%27s_kappa" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t"><span>Talk</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Cohen%27s_kappa"><span>Read</span></a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Cohen%27s_kappa"><span>Read</span></a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Cohen%27s_kappa" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Cohen%27s_kappa" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u"><span>Upload file</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&oldid=1245316837" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Cohen%27s_kappa&id=1245316837&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FCohen%2527s_kappa"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FCohen%2527s_kappa"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Cohen%27s_kappa&action=show-download-screen" title="Download this page as a PDF file"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Cohen%27s_kappa&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q1107106" title="Structured data on this page hosted by Wikidata [g]" accesskey="g"><span>Wikidata item</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Statistic measuring inter-rater agreement for categorical items</div> <p><b>Cohen's kappa coefficient</b> (<i><b>κ</b></i>, lowercase Greek <a href="/wiki/Kappa" title="Kappa">kappa</a>) is a <a href="/wiki/Statistic" title="Statistic">statistic</a> that is used to measure <a href="/wiki/Inter-rater_reliability" title="Inter-rater reliability">inter-rater reliability</a> (and also <a href="/wiki/Intra-rater_reliability" title="Intra-rater reliability">intra-rater reliability</a>) for qualitative (categorical) items.<sup id="cite_ref-Mary2012_1-0" class="reference"><a href="#cite_note-Mary2012-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> It is generally thought to be a more robust measure than simple percent agreement calculation, as <i>κ</i> takes into account the possibility of the agreement occurring by chance. There is controversy surrounding Cohen's kappa due to the difficulty in interpreting indices of agreement. Some researchers have suggested that it is conceptually simpler to evaluate disagreement between items.<sup id="cite_ref-:0_2-0" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p> <meta property="mw:PageProp/toc" /> <div class="mw-heading mw-heading2"><h2 id="History">History</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=1" title="Edit section: History"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The first mention of a kappa-like statistic is attributed to Galton in 1892.<sup id="cite_ref-3" class="reference"><a href="#cite_note-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-4" class="reference"><a href="#cite_note-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup> </p><p>The seminal paper introducing kappa as a new technique was published by <a href="/wiki/Jacob_Cohen_(statistician)" title="Jacob Cohen (statistician)">Jacob Cohen</a> in the journal <i>Educational and Psychological Measurement</i> in 1960.<sup id="cite_ref-5" class="reference"><a href="#cite_note-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Definition">Definition</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=2" title="Edit section: Definition"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Cohen's kappa measures the agreement between two raters who each classify <i>N</i> items into <i>C</i> mutually exclusive categories. The definition of <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \kappa }"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mi>κ</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\textstyle \kappa }</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/526650b8e2dddf1744981effa26203cd83d6cfc2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.339ex; height:1.676ex;" alt="{\textstyle \kappa }"></span> is </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa \equiv {\frac {p_{o}-p_{e}}{1-p_{e}}}=1-{\frac {1-p_{o}}{1-p_{e}}},}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>κ</mi> <mo>≡</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>o</mi> </mrow> </msub> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>1</mn> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>1</mn> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>o</mi> </mrow> </msub> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>,</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa \equiv {\frac {p_{o}-p_{e}}{1-p_{e}}}=1-{\frac {1-p_{o}}{1-p_{e}}},}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1a7a8f738ea187a60443bac93cd53b4d9d6b2231" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:27.267ex; height:5.843ex;" alt="{\displaystyle \kappa \equiv {\frac {p_{o}-p_{e}}{1-p_{e}}}=1-{\frac {1-p_{o}}{1-p_{e}}},}"></span></dd></dl> <p>where <span class="texhtml mvar" style="font-style:italic;">p<sub>o</sub></span> is the relative observed agreement among raters, and <span class="texhtml mvar" style="font-style:italic;">p<sub>e</sub></span> is the hypothetical probability of chance agreement, using the observed data to calculate the probabilities of each observer randomly selecting each category. If the raters are in complete agreement then <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \kappa =1}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mn>1</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\textstyle \kappa =1}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0dbc8882ecebb618c917cd3da3f8d4ec31a15871" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.6ex; height:2.176ex;" alt="{\textstyle \kappa =1}"></span>. If there is no agreement among the raters other than what would be expected by chance (as given by <span class="texhtml mvar" style="font-style:italic;">p<sub>e</sub></span>), <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \kappa =0}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mn>0</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\textstyle \kappa =0}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/7bd9372e70c1d275e6bd1e4979cc42d6274dd51c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.6ex; height:2.176ex;" alt="{\textstyle \kappa =0}"></span>. It is possible for the statistic to be negative,<sup id="cite_ref-6" class="reference"><a href="#cite_note-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> which can occur by chance if there is no relationship between the ratings of the two raters, or it may reflect a real tendency of the raters to give differing ratings. </p><p>For <span class="texhtml mvar" style="font-style:italic;">k</span> categories, <span class="texhtml mvar" style="font-style:italic;">N</span> observations to categorize and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{ki}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n_{ki}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3854e7167ede1b61d0de980a927f2a7e71b73b5e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.051ex; height:2.009ex;" alt="{\displaystyle n_{ki}}"></span> the number of times rater <span class="texhtml mvar" style="font-style:italic;">i</span> predicted category <span class="texhtml mvar" style="font-style:italic;">k</span>: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{e}={\frac {1}{N^{2}}}\sum _{k}n_{k1}n_{k2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <msup> <mi>N</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mfrac> </mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munder> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{e}={\frac {1}{N^{2}}}\sum _{k}n_{k1}n_{k2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/408205fbf33b446ddca56631d529256e98d36066" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; margin-left: -0.089ex; width:20.109ex; height:6.343ex;" alt="{\displaystyle p_{e}={\frac {1}{N^{2}}}\sum _{k}n_{k1}n_{k2}}"></span></dd></dl> <p>This is derived from the following construction: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{e}=\sum _{k}{\widehat {p_{k12}}}{\overset {\text{ind.}}{=}}\sum _{k}{\widehat {p_{k1}}}{\widehat {p_{k2}}}=\sum _{k}{\frac {n_{k1}}{N}}{\frac {n_{k2}}{N}}={\frac {1}{N^{2}}}\sum _{k}n_{k1}n_{k2}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>12</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mover> <mo>=</mo> <mtext>ind.</mtext> </mover> </mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>2</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mi>N</mi> </mfrac> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>2</mn> </mrow> </msub> <mi>N</mi> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mn>1</mn> <msup> <mi>N</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mfrac> </mrow> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munder> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>2</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{e}=\sum _{k}{\widehat {p_{k12}}}{\overset {\text{ind.}}{=}}\sum _{k}{\widehat {p_{k1}}}{\widehat {p_{k2}}}=\sum _{k}{\frac {n_{k1}}{N}}{\frac {n_{k2}}{N}}={\frac {1}{N^{2}}}\sum _{k}n_{k1}n_{k2}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/70ebfa9b030ba3594c404cd585636862546db832" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; margin-left: -0.089ex; width:59.007ex; height:6.343ex;" alt="{\displaystyle p_{e}=\sum _{k}{\widehat {p_{k12}}}{\overset {\text{ind.}}{=}}\sum _{k}{\widehat {p_{k1}}}{\widehat {p_{k2}}}=\sum _{k}{\frac {n_{k1}}{N}}{\frac {n_{k2}}{N}}={\frac {1}{N^{2}}}\sum _{k}n_{k1}n_{k2}}"></span></dd></dl> <p>Where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\widehat {p_{k12}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>12</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\widehat {p_{k12}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/663bfb4ac78fc254c03576f99db9b1c06d2ae61e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:3.992ex; height:2.676ex;" alt="{\displaystyle {\widehat {p_{k12}}}}"></span> is the estimated probability that both rater 1 and rater 2 will classify the same item as k, while <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\widehat {p_{k1}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\widehat {p_{k1}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/745e52211f0c20cae015b49459ed2349f909bb56" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:3.17ex; height:2.676ex;" alt="{\displaystyle {\widehat {p_{k1}}}}"></span> is the estimated probability that rater 1 will classify an item as k (and similarly for rater 2). The relation <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle {\widehat {p_{k}}}=\sum _{k}{\widehat {p_{k1}}}{\widehat {p_{k2}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="false" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> <mo>=</mo> <munder> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munder> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>2</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\textstyle {\widehat {p_{k}}}=\sum _{k}{\widehat {p_{k1}}}{\widehat {p_{k2}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/0f42ee6c6cdcef83a6af001e61c28df33ad69934" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; margin-left: -0.057ex; width:15.569ex; height:3.009ex;" alt="{\textstyle {\widehat {p_{k}}}=\sum _{k}{\widehat {p_{k1}}}{\widehat {p_{k2}}}}"></span> is based on using the assumption that the rating of the two raters are <a href="/wiki/Independence_(probability_theory)" title="Independence (probability theory)">independent</a>. The term <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\widehat {p_{k1}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\widehat {p_{k1}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/745e52211f0c20cae015b49459ed2349f909bb56" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:3.17ex; height:2.676ex;" alt="{\displaystyle {\widehat {p_{k1}}}}"></span> is estimated by using the number of items classified as k by rater 1 (<span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{k1}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle n_{k1}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/c33dfaee3cb2faf3f94de9e123740e4f515d08ae" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.305ex; height:2.009ex;" alt="{\displaystyle n_{k1}}"></span>) divided by the total items to classify (<span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle N}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>N</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle N}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f5e3890c981ae85503089652feb48b191b57aae3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.064ex; height:2.176ex;" alt="{\displaystyle N}"></span>): <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\widehat {p_{k1}}}={n_{k1} \over N}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mover> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mo>^</mo> </mover> </mrow> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <msub> <mi>n</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> <mn>1</mn> </mrow> </msub> <mi>N</mi> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle {\widehat {p_{k1}}}={n_{k1} \over N}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/79740a96d7b530217628631ecb008131089dd67e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; margin-left: -0.089ex; width:10.41ex; height:4.676ex;" alt="{\displaystyle {\widehat {p_{k1}}}={n_{k1} \over N}}"></span> (and similarly for rater 2). </p> <div class="mw-heading mw-heading3"><h3 id="Binary_classification_confusion_matrix">Binary classification confusion matrix</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=3" title="Edit section: Binary classification confusion matrix"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>In the traditional 2 × 2 <a href="/wiki/Confusion_matrix" title="Confusion matrix">confusion matrix</a> employed in <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a> and <a href="/wiki/Statistics" title="Statistics">statistics</a> to evaluate <a href="/wiki/Binary_classification" title="Binary classification">binary classifications</a>, the Cohen's Kappa formula can be written as:<sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa ={\frac {2\times (TP\times TN-FN\times FP)}{(TP+FP)\times (FP+TN)+(TP+FN)\times (FN+TN)}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>2</mn> <mo>×</mo> <mo stretchy="false">(</mo> <mi>T</mi> <mi>P</mi> <mo>×</mo> <mi>T</mi> <mi>N</mi> <mo>−</mo> <mi>F</mi> <mi>N</mi> <mo>×</mo> <mi>F</mi> <mi>P</mi> <mo stretchy="false">)</mo> </mrow> <mrow> <mo stretchy="false">(</mo> <mi>T</mi> <mi>P</mi> <mo>+</mo> <mi>F</mi> <mi>P</mi> <mo stretchy="false">)</mo> <mo>×</mo> <mo stretchy="false">(</mo> <mi>F</mi> <mi>P</mi> <mo>+</mo> <mi>T</mi> <mi>N</mi> <mo stretchy="false">)</mo> <mo>+</mo> <mo stretchy="false">(</mo> <mi>T</mi> <mi>P</mi> <mo>+</mo> <mi>F</mi> <mi>N</mi> <mo stretchy="false">)</mo> <mo>×</mo> <mo stretchy="false">(</mo> <mi>F</mi> <mi>N</mi> <mo>+</mo> <mi>T</mi> <mi>N</mi> <mo stretchy="false">)</mo> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa ={\frac {2\times (TP\times TN-FN\times FP)}{(TP+FP)\times (FP+TN)+(TP+FN)\times (FN+TN)}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/3047cacda9df4c0f7f02c8cbaa61c6f8d8b63c7c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:61.138ex; height:6.509ex;" alt="{\displaystyle \kappa ={\frac {2\times (TP\times TN-FN\times FP)}{(TP+FP)\times (FP+TN)+(TP+FN)\times (FN+TN)}}}"></span></dd></dl> <p>where TP are the true positives, FP are the false positives, TN are the true negatives, and FN are the false negatives. In this case, Cohen's Kappa is equivalent to the <i>Heidke skill score</i> known in <a href="/wiki/Meteorology" title="Meteorology">Meteorology</a>.<sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> The measure was first introduced by Myrick Haskell Doolittle in 1888.<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Examples">Examples</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=4" title="Edit section: Examples"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Simple_example">Simple example</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=5" title="Edit section: Simple example"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Suppose that you were analyzing data related to a group of 50 people applying for a grant. Each grant proposal was read by two readers and each reader either said "Yes" or "No" to the proposal. Suppose the disagreement count data were as follows, where A and B are readers, data on the main diagonal of the matrix (a and d) count the number of agreements and off-diagonal data (b and c) count the number of disagreements: </p> <table class="wikitable"> <tbody><tr> <th style="background:#EAECF0;background:linear-gradient(to top right,#EAECF0 49%,#AAA 49.5%,#AAA 50.5%,#EAECF0 51%);line-height:1.2;padding:0.1em 0.4em;"><div style="margin-left:2em;text-align:right">B</div><div style="margin-right:2em;text-align:left">A</div> </th> <th>Yes </th> <th>No </th></tr> <tr> <th>Yes </th> <td>a </td> <td>b </td></tr> <tr> <th>No </th> <td>c </td> <td>d </td></tr></tbody></table> <p>e.g. </p> <table class="wikitable"> <tbody><tr> <th style="background:#EAECF0;background:linear-gradient(to top right,#EAECF0 49%,#AAA 49.5%,#AAA 50.5%,#EAECF0 51%);line-height:1.2;padding:0.1em 0.4em;"><div style="margin-left:2em;text-align:right">B</div><div style="margin-right:2em;text-align:left">A</div> </th> <th>Yes </th> <th>No </th></tr> <tr> <th>Yes </th> <td>20 </td> <td>5 </td></tr> <tr> <th>No </th> <td>10 </td> <td>15 </td></tr></tbody></table> <p>The observed proportionate agreement is: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{o}={\frac {a+d}{a+b+c+d}}={\frac {20+15}{50}}=0.7}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>o</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>a</mi> <mo>+</mo> <mi>d</mi> </mrow> <mrow> <mi>a</mi> <mo>+</mo> <mi>b</mi> <mo>+</mo> <mi>c</mi> <mo>+</mo> <mi>d</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>20</mn> <mo>+</mo> <mn>15</mn> </mrow> <mn>50</mn> </mfrac> </mrow> <mo>=</mo> <mn>0.7</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{o}={\frac {a+d}{a+b+c+d}}={\frac {20+15}{50}}=0.7}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/1757fb7677fea46fca2bf2d0924f91219dfe563a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.171ex; margin-left: -0.089ex; width:36.689ex; height:5.676ex;" alt="{\displaystyle p_{o}={\frac {a+d}{a+b+c+d}}={\frac {20+15}{50}}=0.7}"></span></dd></dl> <p>To calculate <span class="texhtml mvar" style="font-style:italic;">p<sub>e</sub></span> (the probability of random agreement) we note that: </p> <ul><li>Reader A said "Yes" to 25 applicants and "No" to 25 applicants. Thus reader A said "Yes" 50% of the time.</li> <li>Reader B said "Yes" to 30 applicants and "No" to 20 applicants. Thus reader B said "Yes" 60% of the time.</li></ul> <p>So the expected probability that both would say yes at random is: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{\text{Yes}}={\frac {a+b}{a+b+c+d}}\cdot {\frac {a+c}{a+b+c+d}}=0.5\times 0.6=0.3}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>Yes</mtext> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>a</mi> <mo>+</mo> <mi>b</mi> </mrow> <mrow> <mi>a</mi> <mo>+</mo> <mi>b</mi> <mo>+</mo> <mi>c</mi> <mo>+</mo> <mi>d</mi> </mrow> </mfrac> </mrow> <mo>⋅</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>a</mi> <mo>+</mo> <mi>c</mi> </mrow> <mrow> <mi>a</mi> <mo>+</mo> <mi>b</mi> <mo>+</mo> <mi>c</mi> <mo>+</mo> <mi>d</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0.5</mn> <mo>×</mo> <mn>0.6</mn> <mo>=</mo> <mn>0.3</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{\text{Yes}}={\frac {a+b}{a+b+c+d}}\cdot {\frac {a+c}{a+b+c+d}}=0.5\times 0.6=0.3}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/14d9bf3d72f58a76d78189d508fd80ac83e5f94e" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.171ex; margin-left: -0.089ex; width:54.447ex; height:5.676ex;" alt="{\displaystyle p_{\text{Yes}}={\frac {a+b}{a+b+c+d}}\cdot {\frac {a+c}{a+b+c+d}}=0.5\times 0.6=0.3}"></span></dd></dl> <p>Similarly: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{\text{No}}={\frac {c+d}{a+b+c+d}}\cdot {\frac {b+d}{a+b+c+d}}=0.5\times 0.4=0.2}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>No</mtext> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>c</mi> <mo>+</mo> <mi>d</mi> </mrow> <mrow> <mi>a</mi> <mo>+</mo> <mi>b</mi> <mo>+</mo> <mi>c</mi> <mo>+</mo> <mi>d</mi> </mrow> </mfrac> </mrow> <mo>⋅</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mi>b</mi> <mo>+</mo> <mi>d</mi> </mrow> <mrow> <mi>a</mi> <mo>+</mo> <mi>b</mi> <mo>+</mo> <mi>c</mi> <mo>+</mo> <mi>d</mi> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0.5</mn> <mo>×</mo> <mn>0.4</mn> <mo>=</mo> <mn>0.2</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{\text{No}}={\frac {c+d}{a+b+c+d}}\cdot {\frac {b+d}{a+b+c+d}}=0.5\times 0.4=0.2}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d8fb5669567629ba2010e67b2a16bb1873cf91e4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.171ex; margin-left: -0.089ex; width:53.891ex; height:5.676ex;" alt="{\displaystyle p_{\text{No}}={\frac {c+d}{a+b+c+d}}\cdot {\frac {b+d}{a+b+c+d}}=0.5\times 0.4=0.2}"></span></dd></dl> <p>Overall random agreement probability is the probability that they agreed on either Yes or No, i.e.: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle p_{e}=p_{\text{Yes}}+p_{\text{No}}=0.3+0.2=0.5}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> <mo>=</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>Yes</mtext> </mrow> </msub> <mo>+</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mtext>No</mtext> </mrow> </msub> <mo>=</mo> <mn>0.3</mn> <mo>+</mo> <mn>0.2</mn> <mo>=</mo> <mn>0.5</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle p_{e}=p_{\text{Yes}}+p_{\text{No}}=0.3+0.2=0.5}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/ea11f70a3b9422816c7d7ff2f7e32bf4f1ca464c" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.089ex; width:33.617ex; height:2.509ex;" alt="{\displaystyle p_{e}=p_{\text{Yes}}+p_{\text{No}}=0.3+0.2=0.5}"></span></dd></dl> <p>So now applying our formula for Cohen's Kappa we get: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa ={\frac {p_{o}-p_{e}}{1-p_{e}}}={\frac {0.7-0.5}{1-0.5}}=0.4}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>o</mi> </mrow> </msub> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> </mrow> </mfrac> </mrow> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>0.7</mn> <mo>−</mo> <mn>0.5</mn> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <mn>0.5</mn> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0.4</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa ={\frac {p_{o}-p_{e}}{1-p_{e}}}={\frac {0.7-0.5}{1-0.5}}=0.4}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/cdfa606ccfe8c12397afcea7348ca067c978f16b" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.338ex; width:31.27ex; height:5.676ex;" alt="{\displaystyle \kappa ={\frac {p_{o}-p_{e}}{1-p_{e}}}={\frac {0.7-0.5}{1-0.5}}=0.4}"></span></dd></dl> <div class="mw-heading mw-heading3"><h3 id="Same_percentages_but_different_numbers">Same percentages but different numbers</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=6" title="Edit section: Same percentages but different numbers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A case sometimes considered to be a problem with Cohen's Kappa occurs when comparing the Kappa calculated for two pairs of raters with the two raters in each pair having the same percentage agreement but one pair give a similar number of ratings in each class while the other pair give a very different number of ratings in each class.<sup id="cite_ref-Gwet2002_10-0" class="reference"><a href="#cite_note-Gwet2002-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> (In the cases below, notice B has 70 yeses and 30 nos, in the first case, but those numbers are reversed in the second.) For instance, in the following two cases there is equal agreement between A and B (60 out of 100 in both cases) in terms of agreement in each class, so we would expect the relative values of Cohen's Kappa to reflect this. However, calculating Cohen's Kappa for each: </p> <table class="wikitable"> <tbody><tr> <th style="background:#EAECF0;background:linear-gradient(to top right,#EAECF0 49%,#AAA 49.5%,#AAA 50.5%,#EAECF0 51%);line-height:1.2;padding:0.1em 0.4em;"><div style="margin-left:2em;text-align:right">B</div><div style="margin-right:2em;text-align:left">A</div> </th> <th>Yes </th> <th>No </th></tr> <tr> <th>Yes </th> <td>45 </td> <td>15 </td></tr> <tr> <th>No </th> <td>25 </td> <td>15 </td></tr></tbody></table> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa ={\frac {0.60-0.54}{1-0.54}}=0.1304}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>0.60</mn> <mo>−</mo> <mn>0.54</mn> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <mn>0.54</mn> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0.1304</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa ={\frac {0.60-0.54}{1-0.54}}=0.1304}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/331efc08313193e518f410dbaca1699283e08803" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.005ex; width:25.94ex; height:5.343ex;" alt="{\displaystyle \kappa ={\frac {0.60-0.54}{1-0.54}}=0.1304}"></span></dd></dl> <table class="wikitable"> <tbody><tr> <th style="background:#EAECF0;background:linear-gradient(to top right,#EAECF0 49%,#AAA 49.5%,#AAA 50.5%,#EAECF0 51%);line-height:1.2;padding:0.1em 0.4em;"><div style="margin-left:2em;text-align:right">B</div><div style="margin-right:2em;text-align:left">A</div> </th> <th>Yes </th> <th>No </th></tr> <tr> <th>Yes </th> <td>25 </td> <td>35 </td></tr> <tr> <th>No </th> <td>5 </td> <td>35 </td></tr></tbody></table> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa ={\frac {0.60-0.46}{1-0.46}}=0.2593}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <mn>0.60</mn> <mo>−</mo> <mn>0.46</mn> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <mn>0.46</mn> </mrow> </mfrac> </mrow> <mo>=</mo> <mn>0.2593</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa ={\frac {0.60-0.46}{1-0.46}}=0.2593}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b30abe3a577420bb3706afd7c659be8f073fefa3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.005ex; width:25.94ex; height:5.343ex;" alt="{\displaystyle \kappa ={\frac {0.60-0.46}{1-0.46}}=0.2593}"></span></dd></dl> <p>we find that it shows greater similarity between A and B in the second case, compared to the first. This is because while the percentage agreement is the same, the percentage agreement that would occur 'by chance' is significantly higher in the first case (0.54 compared to 0.46). </p> <div class="mw-heading mw-heading2"><h2 id="Properties">Properties</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=7" title="Edit section: Properties"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Hypothesis_testing_and_confidence_interval">Hypothesis testing and confidence interval</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=8" title="Edit section: Hypothesis testing and confidence interval"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p><a href="/wiki/P-value" title="P-value">P-value</a> for kappa is rarely reported, probably because even relatively low values of kappa can nonetheless be significantly different from zero but not of sufficient magnitude to satisfy investigators.<sup id="cite_ref-BakemanGottman1997_11-0" class="reference"><a href="#cite_note-BakemanGottman1997-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 66">: 66 </span></sup> Still, its standard error has been described<sup id="cite_ref-FleissCohenEv1969_12-0" class="reference"><a href="#cite_note-FleissCohenEv1969-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> and is computed by various computer programs.<sup id="cite_ref-BakemanRobinson1998_13-0" class="reference"><a href="#cite_note-BakemanRobinson1998-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> </p><p><a href="/wiki/Confidence_intervals" class="mw-redirect" title="Confidence intervals">Confidence intervals</a> for Kappa may be constructed, for the expected Kappa values if we had infinite number of items checked, using the following formula:<sup id="cite_ref-Mary2012_1-1" class="reference"><a href="#cite_note-Mary2012-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle CI:\kappa \pm Z_{1-\alpha /2}SE_{\kappa }}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>C</mi> <mi>I</mi> <mo>:</mo> <mi>κ</mi> <mo>±</mo> <msub> <mi>Z</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>−</mo> <mi>α</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mn>2</mn> </mrow> </msub> <mi>S</mi> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>κ</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle CI:\kappa \pm Z_{1-\alpha /2}SE_{\kappa }}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f937f35e9b8e5ac56f9c0b4260069d61582641c3" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:20.064ex; height:3.009ex;" alt="{\displaystyle CI:\kappa \pm Z_{1-\alpha /2}SE_{\kappa }}"></span></dd></dl> <p>Where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Z_{1-\alpha /2}=1.960}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>Z</mi> <mrow class="MJX-TeXAtom-ORD"> <mn>1</mn> <mo>−</mo> <mi>α</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>/</mo> </mrow> <mn>2</mn> </mrow> </msub> <mo>=</mo> <mn>1.960</mn> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle Z_{1-\alpha /2}=1.960}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/153ef6004c7335c6b6c2ca165b1b166e89aa10b4" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:15.011ex; height:3.009ex;" alt="{\displaystyle Z_{1-\alpha /2}=1.960}"></span> is the <a href="/wiki/Standard_normal" class="mw-redirect" title="Standard normal">standard normal</a> percentile when <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha =5\%}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>α</mi> <mo>=</mo> <mn>5</mn> <mi mathvariant="normal">%</mi> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \alpha =5\%}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/b3a8efe77507eb509f799e62518d425e544590a2" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.684ex; height:2.343ex;" alt="{\displaystyle \alpha =5\%}"></span>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle SE_{\kappa }={\sqrt {{p_{o}(1-p_{o})} \over {N(1-p_{e})^{2}}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>S</mi> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>κ</mi> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <msqrt> <mfrac> <mrow class="MJX-TeXAtom-ORD"> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>o</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>o</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>N</mi> <mo stretchy="false">(</mo> <mn>1</mn> <mo>−</mo> <msub> <mi>p</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>e</mi> </mrow> </msub> <msup> <mo stretchy="false">)</mo> <mrow class="MJX-TeXAtom-ORD"> <mn>2</mn> </mrow> </msup> </mrow> </mfrac> </msqrt> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle SE_{\kappa }={\sqrt {{p_{o}(1-p_{o})} \over {N(1-p_{e})^{2}}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/46a9eb3935acc587aa69549db7c7cc8f262e2199" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:21.75ex; height:7.509ex;" alt="{\displaystyle SE_{\kappa }={\sqrt {{p_{o}(1-p_{o})} \over {N(1-p_{e})^{2}}}}}"></span> </p><p>This is calculated by ignoring that <span class="texhtml mvar" style="font-style:italic;">p<sub>e</sub></span> is estimated from the data, and by treating <span class="texhtml mvar" style="font-style:italic;">p<sub>o</sub></span> as an estimated probability of a <a href="/wiki/Binomial_distribution" title="Binomial distribution">binomial distribution</a> while using asymptotic normality (i.e.: assuming that the number of items is large and that <span class="texhtml mvar" style="font-style:italic;">p<sub>o</sub></span> is not close to either 0 or 1). <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle SE_{\kappa }}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>S</mi> <msub> <mi>E</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>κ</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle SE_{\kappa }}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/df914f6bf5f39b8116504ddd6643083a8b390c47" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.394ex; height:2.509ex;" alt="{\displaystyle SE_{\kappa }}"></span> (and the CI in general) may also be estimated using <a href="/wiki/Bootstrapping_(statistics)" title="Bootstrapping (statistics)">bootstrap</a> methods. </p> <div class="mw-heading mw-heading3"><h3 id="Interpreting_magnitude">Interpreting magnitude</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=9" title="Edit section: Interpreting magnitude"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure class="mw-halign-right" typeof="mw:File/Thumb"><a href="/wiki/File:Kappa_vs_accuracy.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/en/thumb/9/92/Kappa_vs_accuracy.png/400px-Kappa_vs_accuracy.png" decoding="async" width="400" height="281" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/9/92/Kappa_vs_accuracy.png/600px-Kappa_vs_accuracy.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/9/92/Kappa_vs_accuracy.png/800px-Kappa_vs_accuracy.png 2x" data-file-width="1109" data-file-height="780" /></a><figcaption>Kappa (vertical axis) and <a href="/wiki/Accuracy_and_precision#In_binary_classification" title="Accuracy and precision">Accuracy</a> (horizontal axis) calculated from the same simulated binary data. Each point on the graph is calculated from a pairs of judges randomly rating 10 subjects for having a diagnosis of X or not. Note in this example a Kappa=0 is approximately equivalent to an accuracy=0.5</figcaption></figure> <p>If statistical significance is not a useful guide, what magnitude of kappa reflects adequate agreement? Guidelines would be helpful, but factors other than agreement can influence its magnitude, which makes interpretation of a given magnitude problematic. As Sim and Wright noted, two important factors are prevalence (are the codes equiprobable or do their probabilities vary) and bias (are the marginal probabilities for the two observers similar or different). Other things being equal, kappas are higher when codes are equiprobable. On the other hand, Kappas are higher when codes are distributed asymmetrically by the two observers. In contrast to probability variations, the effect of bias is greater when Kappa is small than when it is large.<sup id="cite_ref-SimWright2005_14-0" class="reference"><a href="#cite_note-SimWright2005-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 261–262">: 261–262 </span></sup> </p><p>Another factor is the number of codes. As number of codes increases, kappas become higher. Based on a simulation study, Bakeman and colleagues concluded that for fallible observers, values for kappa were lower when codes were fewer. And, in agreement with Sim & Wrights's statement concerning prevalence, kappas were higher when codes were roughly equiprobable. Thus Bakeman et al. concluded that "no one value of kappa can be regarded as universally acceptable."<sup id="cite_ref-BakemanEtAl1997_15-0" class="reference"><a href="#cite_note-BakemanEtAl1997-15"><span class="cite-bracket">[</span>15<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 357">: 357 </span></sup> They also provide a computer program that lets users compute values for kappa specifying number of codes, their probability, and observer accuracy. For example, given equiprobable codes and observers who are 85% accurate, value of kappa are 0.49, 0.60, 0.66, and 0.69 when number of codes is 2, 3, 5, and 10, respectively. </p><p>Nonetheless, magnitude guidelines have appeared in the literature. Perhaps the first was Landis and Koch,<sup id="cite_ref-LandisKoch1977_16-0" class="reference"><a href="#cite_note-LandisKoch1977-16"><span class="cite-bracket">[</span>16<span class="cite-bracket">]</span></a></sup> who characterized values < 0 as indicating no agreement and 0–0.20 as slight, 0.21–0.40 as fair, 0.41–0.60 as moderate, 0.61–0.80 as substantial, and 0.81–1 as almost perfect agreement. This set of guidelines is however by no means universally accepted; Landis and Koch supplied no evidence to support it, basing it instead on personal opinion. It has been noted that these guidelines may be more harmful than helpful.<sup id="cite_ref-17" class="reference"><a href="#cite_note-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup> Fleiss's<sup id="cite_ref-Fleiss1981_18-0" class="reference"><a href="#cite_note-Fleiss1981-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 218">: 218 </span></sup> equally arbitrary guidelines characterize kappas over 0.75 as excellent, 0.40 to 0.75 as fair to good, and below 0.40 as poor. </p> <div class="mw-heading mw-heading3"><h3 id="Kappa_maximum">Kappa maximum</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=10" title="Edit section: Kappa maximum"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Kappa assumes its theoretical maximum value of 1 only when both observers distribute codes the same, that is, when corresponding row and column sums are identical. Anything less is less than perfect agreement. Still, the maximum value kappa could achieve given unequal distributions helps interpret the value of kappa actually obtained. The equation for <i>κ</i> maximum is:<sup id="cite_ref-Umesh989_19-0" class="reference"><a href="#cite_note-Umesh989-19"><span class="cite-bracket">[</span>19<span class="cite-bracket">]</span></a></sup> </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa _{\max }={\frac {P_{\max }-P_{\exp }}{1-P_{\exp }}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>κ</mi> <mrow class="MJX-TeXAtom-ORD"> <mo movablelimits="true" form="prefix">max</mo> </mrow> </msub> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo movablelimits="true" form="prefix">max</mo> </mrow> </msub> <mo>−</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>exp</mi> </mrow> </msub> </mrow> <mrow> <mn>1</mn> <mo>−</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>exp</mi> </mrow> </msub> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa _{\max }={\frac {P_{\max }-P_{\exp }}{1-P_{\exp }}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f1a31e80d07acb8ca2f1c733010d8498c8791551" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:20.425ex; height:6.176ex;" alt="{\displaystyle \kappa _{\max }={\frac {P_{\max }-P_{\exp }}{1-P_{\exp }}}}"></span></dd></dl> <p>where <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{\exp }=\sum _{i=1}^{k}P_{i+}P_{+i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>exp</mi> </mrow> </msub> <mo>=</mo> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munderover> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>+</mo> </mrow> </msub> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>+</mo> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{\exp }=\sum _{i=1}^{k}P_{i+}P_{+i}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/97df0fdf3d7da1313f4e6464fd606f7c2dba7b68" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:18.218ex; height:7.343ex;" alt="{\displaystyle P_{\exp }=\sum _{i=1}^{k}P_{i+}P_{+i}}"></span>, as usual, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{\max }=\sum _{i=1}^{k}\min(P_{i+},P_{+i})}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo movablelimits="true" form="prefix">max</mo> </mrow> </msub> <mo>=</mo> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munderover> <mo movablelimits="true" form="prefix">min</mo> <mo stretchy="false">(</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>+</mo> </mrow> </msub> <mo>,</mo> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>+</mo> <mi>i</mi> </mrow> </msub> <mo stretchy="false">)</mo> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{\max }=\sum _{i=1}^{k}\min(P_{i+},P_{+i})}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/f28177cb0903390a3160f023e309244f2c773050" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:25.483ex; height:7.343ex;" alt="{\displaystyle P_{\max }=\sum _{i=1}^{k}\min(P_{i+},P_{+i})}"></span>, </p><p><i>k</i> = number of codes, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{i+}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>+</mo> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{i+}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/6b9dfd58ff73aeae22b559d5b295374f4148f146" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.57ex; height:2.509ex;" alt="{\displaystyle P_{i+}}"></span> are the row probabilities, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle P_{+i}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>P</mi> <mrow class="MJX-TeXAtom-ORD"> <mo>+</mo> <mi>i</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle P_{+i}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/fa219ddc211e776fd0f6aa1efe557279c3422cab" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.57ex; height:2.509ex;" alt="{\displaystyle P_{+i}}"></span> are the column probabilities. </p> <div class="mw-heading mw-heading3"><h3 id="Limitations">Limitations</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=11" title="Edit section: Limitations"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Kappa is an index that considers observed agreement with respect to a baseline agreement. However, investigators must consider carefully whether Kappa's baseline agreement is relevant for the particular research question. Kappa's baseline is frequently described as the agreement due to chance, which is only partially correct. Kappa's baseline agreement is the agreement that would be expected due to random allocation, given the quantities specified by the marginal totals of square contingency table. Thus, κ = 0 when the observed allocation is apparently random, regardless of the quantity disagreement as constrained by the marginal totals. However, for many applications, investigators should be more interested in the quantity disagreement in the marginal totals than in the allocation disagreement as described by the additional information on the diagonal of the square contingency table. Thus for many applications, Kappa's baseline is more distracting than enlightening. Consider the following example: </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Kappa_example.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/en/thumb/1/1d/Kappa_example.png/220px-Kappa_example.png" decoding="async" width="220" height="165" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/1/1d/Kappa_example.png/330px-Kappa_example.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/1/1d/Kappa_example.png/440px-Kappa_example.png 2x" data-file-width="2133" data-file-height="1600" /></a><figcaption>Kappa example</figcaption></figure> <table class="wikitable"> <caption>Comparison 1 </caption> <tbody><tr> <th colspan="2" rowspan="2"> </th> <th colspan="2">Reference </th></tr> <tr> <td>G </td> <td>R </td></tr> <tr> <th rowspan="2">Comparison </th> <td>G </td> <td>1 </td> <td>14 </td></tr> <tr> <td>R </td> <td>0 </td> <td>1 </td></tr></tbody></table> <p>The disagreement proportion is 14/16 or 0.875. The disagreement is due to quantity because allocation is optimal. κ is 0.01. </p> <table class="wikitable"> <caption>Comparison 2 </caption> <tbody><tr> <th colspan="2" rowspan="2"> </th> <th colspan="2">Reference </th></tr> <tr> <td>G </td> <td>R </td></tr> <tr> <th rowspan="2">Comparison </th> <td>G </td> <td>0 </td> <td>1 </td></tr> <tr> <td>R </td> <td>1 </td> <td>14 </td></tr></tbody></table> <p>The disagreement proportion is 2/16 or 0.125. The disagreement is due to allocation because quantities are identical. Kappa is −0.07. </p><p> Here, reporting quantity and allocation disagreement is informative while Kappa obscures information. Furthermore, Kappa introduces some challenges in calculation and interpretation because Kappa is a ratio. It is possible for Kappa's ratio to return an undefined value due to zero in the denominator. Furthermore, a ratio does not reveal its numerator nor its denominator. It is more informative for researchers to report disagreement in two components, quantity and allocation. These two components describe the relationship between the categories more clearly than a single summary statistic. When predictive accuracy is the goal, researchers can more easily begin to think about ways to improve a prediction by using two components of quantity and allocation, rather than one ratio of Kappa.<sup id="cite_ref-:0_2-1" class="reference"><a href="#cite_note-:0-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup><style data-mw-deduplicate="TemplateStyles:r1236090951">.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}</style></p><div role="note" class="hatnote navigation-not-searchable">For a measure of difference between two continuous variables, see <a href="/wiki/Coefficient_of_determination" title="Coefficient of determination">coefficient of determination</a>.</div><p>Some researchers have expressed concern over κ's tendency to take the observed categories' frequencies as givens, which can make it unreliable for measuring agreement in situations such as the diagnosis of rare diseases. In these situations, κ tends to underestimate the agreement on the rare category.<sup id="cite_ref-20" class="reference"><a href="#cite_note-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup> For this reason, κ is considered an overly conservative measure of agreement.<sup id="cite_ref-SMPJ_21-0" class="reference"><a href="#cite_note-SMPJ-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup> Others<sup id="cite_ref-22" class="reference"><a href="#cite_note-22"><span class="cite-bracket">[</span>22<span class="cite-bracket">]</span></a></sup><sup class="noprint Inline-Template Template-Fact" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed"><span title="This claim needs references to reliable sources. (April 2012)">citation needed</span></a></i>]</sup> contest the assertion that kappa "takes into account" chance agreement. To do this effectively would require an explicit model of how chance affects rater decisions. The so-called chance adjustment of kappa statistics supposes that, when not completely certain, raters simply guess—a very unrealistic scenario. Moreover, some works<sup id="cite_ref-23" class="reference"><a href="#cite_note-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup> have shown how kappa statistics can lead to a wrong conclusion for unbalanced data. </p><div class="mw-heading mw-heading2"><h2 id="Related_statistics">Related statistics</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=12" title="Edit section: Related statistics"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Scott's_Pi"><span id="Scott.27s_Pi"></span>Scott's Pi</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=13" title="Edit section: Scott's Pi"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A similar statistic, called <a href="/wiki/Scott%27s_Pi" title="Scott's Pi">pi</a>, was proposed by Scott (1955). Cohen's kappa and <a href="/wiki/Scott%27s_Pi" title="Scott's Pi">Scott's pi</a> differ in terms of how <span class="texhtml mvar" style="font-style:italic;">p<sub>e</sub></span> is calculated. </p> <div class="mw-heading mw-heading3"><h3 id="Fleiss'_kappa"><span id="Fleiss.27_kappa"></span>Fleiss' kappa</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=14" title="Edit section: Fleiss' kappa"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Note that Cohen's kappa measures agreement between <b>two</b> raters only. For a similar measure of agreement (<a href="/wiki/Fleiss%27_kappa" title="Fleiss' kappa">Fleiss' kappa</a>) used when there are more than two raters, see <a href="/wiki/Joseph_L._Fleiss" title="Joseph L. Fleiss">Fleiss</a> (1971). The Fleiss kappa, however, is a multi-rater generalization of <a href="/wiki/Scott%27s_Pi" title="Scott's Pi">Scott's pi</a> statistic, not Cohen's kappa. Kappa is also used to compare performance in <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a>, but the directional version known as <a href="/wiki/Informedness" class="mw-redirect" title="Informedness">Informedness</a> or <a href="/wiki/Youden%27s_J_statistic" title="Youden's J statistic">Youden's J statistic</a> is argued to be more appropriate for supervised learning.<sup id="cite_ref-Powers2012_24-0" class="reference"><a href="#cite_note-Powers2012-24"><span class="cite-bracket">[</span>24<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Weighted_kappa">Weighted kappa</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=15" title="Edit section: Weighted kappa"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The weighted kappa allows disagreements to be weighted differently<sup id="cite_ref-Cohen1968_25-0" class="reference"><a href="#cite_note-Cohen1968-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup> and is especially useful when codes are ordered.<sup id="cite_ref-BakemanGottman1997_11-1" class="reference"><a href="#cite_note-BakemanGottman1997-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page / location: 66">: 66 </span></sup> Three matrices are involved, the matrix of observed scores, the matrix of expected scores based on chance agreement, and the weight matrix. Weight matrix cells located on the diagonal (upper-left to bottom-right) represent agreement and thus contain zeros. Off-diagonal cells contain weights indicating the seriousness of that disagreement. Often, cells one off the diagonal are weighted 1, those two off 2, etc. </p><p>The equation for weighted κ is: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \kappa =1-{\frac {\sum _{i=1}^{k}\sum _{j=1}^{k}w_{ij}x_{ij}}{\sum _{i=1}^{k}\sum _{j=1}^{k}w_{ij}m_{ij}}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <mi>κ</mi> <mo>=</mo> <mn>1</mn> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mfrac> <mrow> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munderover> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munderover> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> </mrow> <mrow> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munderover> <munderover> <mo>∑</mo> <mrow class="MJX-TeXAtom-ORD"> <mi>j</mi> <mo>=</mo> <mn>1</mn> </mrow> <mrow class="MJX-TeXAtom-ORD"> <mi>k</mi> </mrow> </munderover> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> <msub> <mi>m</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> </mrow> </mfrac> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle \kappa =1-{\frac {\sum _{i=1}^{k}\sum _{j=1}^{k}w_{ij}x_{ij}}{\sum _{i=1}^{k}\sum _{j=1}^{k}w_{ij}m_{ij}}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/2a496e1cef7d812b83bdbb725d291748cf0183f5" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:27.527ex; height:8.176ex;" alt="{\displaystyle \kappa =1-{\frac {\sum _{i=1}^{k}\sum _{j=1}^{k}w_{ij}x_{ij}}{\sum _{i=1}^{k}\sum _{j=1}^{k}w_{ij}m_{ij}}}}"></span></dd></dl> <p>where <i>k</i>=number of codes and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{ij}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>w</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle w_{ij}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d3302ff355269436b43bc2fbe180303881c09321" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.141ex; height:2.343ex;" alt="{\displaystyle w_{ij}}"></span>, <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x_{ij}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>x</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle x_{ij}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/a27e97949cb2cc8f2d4c2a9421477a65f839db11" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.807ex; height:2.343ex;" alt="{\displaystyle x_{ij}}"></span>, and <span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m_{ij}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>m</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>i</mi> <mi>j</mi> </mrow> </msub> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle m_{ij}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/cd6f1bb2d6548dca472922bcbcb77e7ad3a5b4df" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.518ex; height:2.343ex;" alt="{\displaystyle m_{ij}}"></span> are elements in the weight, observed, and expected matrices, respectively. When diagonal cells contain weights of 0 and all off-diagonal cells weights of 1, this formula produces the same value of kappa as the calculation given above. </p> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=16" title="Edit section: See also"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Bangdiwala%27s_B" title="Bangdiwala's B">Bangdiwala's B</a></li> <li><a href="/wiki/Intraclass_correlation" title="Intraclass correlation">Intraclass correlation</a></li> <li><a href="/wiki/Krippendorff%27s_alpha" title="Krippendorff's alpha">Krippendorff's alpha</a></li> <li><a href="/wiki/Statistical_classification" title="Statistical classification">Statistical classification</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=17" title="Edit section: Further reading"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite id="CITEREFBanerjeeCapozzoliMcSweeneySinha1999" class="citation journal cs1">Banerjee, M.; Capozzoli, Michelle; McSweeney, Laura; Sinha, Debajyoti (1999). <a rel="nofollow" class="external text" href="https://digitalcommons.fairfield.edu/mathandcomputerscience-facultypubs/60">"Beyond Kappa: A Review of Interrater Agreement Measures"</a>. <i>The Canadian Journal of Statistics</i>. <b>27</b> (1): 3–23. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.2307%2F3315487">10.2307/3315487</a>. <a href="/wiki/JSTOR_(identifier)" class="mw-redirect" title="JSTOR (identifier)">JSTOR</a> <a rel="nofollow" class="external text" href="https://www.jstor.org/stable/3315487">3315487</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:37082712">37082712</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=The+Canadian+Journal+of+Statistics&rft.atitle=Beyond+Kappa%3A+A+Review+of+Interrater+Agreement+Measures&rft.volume=27&rft.issue=1&rft.pages=3-23&rft.date=1999&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A37082712%23id-name%3DS2CID&rft_id=https%3A%2F%2Fwww.jstor.org%2Fstable%2F3315487%23id-name%3DJSTOR&rft_id=info%3Adoi%2F10.2307%2F3315487&rft.aulast=Banerjee&rft.aufirst=M.&rft.au=Capozzoli%2C+Michelle&rft.au=McSweeney%2C+Laura&rft.au=Sinha%2C+Debajyoti&rft_id=https%3A%2F%2Fdigitalcommons.fairfield.edu%2Fmathandcomputerscience-facultypubs%2F60&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChiccoWarrensJurman2021" class="citation journal cs1">Chicco, D.; Warrens, M.J.; Jurman, G. (2021). <a rel="nofollow" class="external text" href="https://doi.org/10.1109%2Faccess.2021.3084050">"The Matthews correlation coefficient (MCC) is more informative than Cohen's Kappa and Brier score in binary classification assessment"</a>. <i>IEEE Access</i>. <b>9</b>: 78368–81. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2021IEEEA...978368C">2021IEEEA...978368C</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1109%2Faccess.2021.3084050">10.1109/access.2021.3084050</a></span>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/10281%2F430460">10281/430460</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:235308708">235308708</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Access&rft.atitle=The+Matthews+correlation+coefficient+%28MCC%29+is+more+informative+than+Cohen%27s+Kappa+and+Brier+score+in+binary+classification+assessment&rft.volume=9&rft.pages=78368-81&rft.date=2021&rft_id=info%3Ahdl%2F10281%2F430460&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A235308708%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1109%2Faccess.2021.3084050&rft_id=info%3Abibcode%2F2021IEEEA...978368C&rft.aulast=Chicco&rft.aufirst=D.&rft.au=Warrens%2C+M.J.&rft.au=Jurman%2C+G.&rft_id=https%3A%2F%2Fdoi.org%2F10.1109%252Faccess.2021.3084050&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCohen1960" class="citation journal cs1">Cohen, Jacob (1960). "A coefficient of agreement for nominal scales". <i>Educational and Psychological Measurement</i>. <b>20</b> (1): 37–46. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F001316446002000104">10.1177/001316446002000104</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/1942%2F28116">1942/28116</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15926286">15926286</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Educational+and+Psychological+Measurement&rft.atitle=A+coefficient+of+agreement+for+nominal+scales&rft.volume=20&rft.issue=1&rft.pages=37-46&rft.date=1960&rft_id=info%3Ahdl%2F1942%2F28116&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15926286%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1177%2F001316446002000104&rft.aulast=Cohen&rft.aufirst=Jacob&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCohen1968" class="citation journal cs1">Cohen, J. (1968). "Weighted kappa: Nominal scale agreement with provision for scaled disagreement or partial credit". <i>Psychological Bulletin</i>. <b>70</b> (4): 213–220. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1037%2Fh0026256">10.1037/h0026256</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/19673146">19673146</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Psychological+Bulletin&rft.atitle=Weighted+kappa%3A+Nominal+scale+agreement+with+provision+for+scaled+disagreement+or+partial+credit&rft.volume=70&rft.issue=4&rft.pages=213-220&rft.date=1968&rft_id=info%3Adoi%2F10.1037%2Fh0026256&rft_id=info%3Apmid%2F19673146&rft.aulast=Cohen&rft.aufirst=J.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFFleissCohen1973" class="citation journal cs1">Fleiss, J.L.; Cohen, J. (1973). "The equivalence of weighted kappa and the intraclass correlation coefficient as measures of reliability". <i>Educational and Psychological Measurement</i>. <b>33</b> (3): 613–9. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F001316447303300309">10.1177/001316447303300309</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:145183399">145183399</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Educational+and+Psychological+Measurement&rft.atitle=The+equivalence+of+weighted+kappa+and+the+intraclass+correlation+coefficient+as+measures+of+reliability&rft.volume=33&rft.issue=3&rft.pages=613-9&rft.date=1973&rft_id=info%3Adoi%2F10.1177%2F001316447303300309&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A145183399%23id-name%3DS2CID&rft.aulast=Fleiss&rft.aufirst=J.L.&rft.au=Cohen%2C+J.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSimWright2005" class="citation journal cs1">Sim, J.; Wright, C.C. (2005). <a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fptj%2F85.3.257">"The Kappa Statistic in Reliability Studies: Use, Interpretation, and Sample Size Requirements"</a>. <i>Physical Therapy</i>. <b>85</b> (3): 257–268. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fptj%2F85.3.257">10.1093/ptj/85.3.257</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/15733050">15733050</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Therapy&rft.atitle=The+Kappa+Statistic+in+Reliability+Studies%3A+Use%2C+Interpretation%2C+and+Sample+Size+Requirements&rft.volume=85&rft.issue=3&rft.pages=257-268&rft.date=2005&rft_id=info%3Adoi%2F10.1093%2Fptj%2F85.3.257&rft_id=info%3Apmid%2F15733050&rft.aulast=Sim&rft.aufirst=J.&rft.au=Wright%2C+C.C.&rft_id=https%3A%2F%2Fdoi.org%2F10.1093%252Fptj%252F85.3.257&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li> <li><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWarrens2011" class="citation journal cs1">Warrens, J. (2011). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.stamet.2011.06.002">"Cohen's kappa is a weighted average"</a>. <i>Statistical Methodology</i>. <b>8</b> (6): 473–484. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.stamet.2011.06.002">10.1016/j.stamet.2011.06.002</a></span>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/1887%2F18062">1887/18062</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Statistical+Methodology&rft.atitle=Cohen%27s+kappa+is+a+weighted+average&rft.volume=8&rft.issue=6&rft.pages=473-484&rft.date=2011&rft_id=info%3Ahdl%2F1887%2F18062&rft_id=info%3Adoi%2F10.1016%2Fj.stamet.2011.06.002&rft.aulast=Warrens&rft.aufirst=J.&rft_id=https%3A%2F%2Fdoi.org%2F10.1016%252Fj.stamet.2011.06.002&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></li></ul> <div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=18" title="Edit section: External links"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a rel="nofollow" class="external text" href="http://www.agreestat.com/research_papers.html">Kappa, its meaning, problems, and several alternatives</a> (Link is dead as of 2022-12-16)</li> <li><a rel="nofollow" class="external text" href="http://www.john-uebersax.com/stat/kappa.htm#procon">Kappa Statistics: Pros and Cons</a></li> <li>Software implementations <ul><li><a rel="nofollow" class="external text" href="http://www.gsu.edu/~psyrab/ComKappa2.zip">Windows program "ComKappa" for kappa, weighted kappa, and kappa maximum</a> (Error "Access Denied (Error Code 1020)" as of 2022-12-16)</li></ul></li></ul> <div class="mw-heading mw-heading2"><h2 id="References">References</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Cohen%27s_kappa&action=edit&section=19" title="Edit section: References"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em;"> <ol class="references"> <li id="cite_note-Mary2012-1"><span class="mw-cite-backlink">^ <a href="#cite_ref-Mary2012_1-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-Mary2012_1-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMcHugh2012" class="citation journal cs1">McHugh, Mary L. (2012). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3900052">"Interrater reliability: The kappa statistic"</a>. <i>Biochemia Medica</i>. <b>22</b> (3): 276–282. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.11613%2Fbm.2012.031">10.11613/bm.2012.031</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3900052">3900052</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/23092060">23092060</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Biochemia+Medica&rft.atitle=Interrater+reliability%3A+The+kappa+statistic&rft.volume=22&rft.issue=3&rft.pages=276-282&rft.date=2012&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC3900052%23id-name%3DPMC&rft_id=info%3Apmid%2F23092060&rft_id=info%3Adoi%2F10.11613%2Fbm.2012.031&rft.aulast=McHugh&rft.aufirst=Mary+L.&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC3900052&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-:0-2"><span class="mw-cite-backlink">^ <a href="#cite_ref-:0_2-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:0_2-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPontiusMillones2011" class="citation journal cs1">Pontius, Robert; Millones, Marco (2011). <a rel="nofollow" class="external text" href="http://www.clarku.edu/~rpontius/">"Death to Kappa: birth of quantity disagreement and allocation disagreement for accuracy assessment"</a>. <i>International Journal of Remote Sensing</i>. <b>32</b> (15): 4407–4429. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2011IJRS...32.4407P">2011IJRS...32.4407P</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F01431161.2011.552923">10.1080/01431161.2011.552923</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:62883674">62883674</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=International+Journal+of+Remote+Sensing&rft.atitle=Death+to+Kappa%3A+birth+of+quantity+disagreement+and+allocation+disagreement+for+accuracy+assessment&rft.volume=32&rft.issue=15&rft.pages=4407-4429&rft.date=2011&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A62883674%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1080%2F01431161.2011.552923&rft_id=info%3Abibcode%2F2011IJRS...32.4407P&rft.aulast=Pontius&rft.aufirst=Robert&rft.au=Millones%2C+Marco&rft_id=http%3A%2F%2Fwww.clarku.edu%2F~rpontius%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-3"><span class="mw-cite-backlink"><b><a href="#cite_ref-3">^</a></b></span> <span class="reference-text">Galton, F. (1892) <i><a href="/wiki/Finger_Prints_(book)" title="Finger Prints (book)">Finger Prints</a></i> Macmillan, London.</span> </li> <li id="cite_note-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-4">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSmeeton1985" class="citation journal cs1">Smeeton, N.C. (1985). "Early History of the Kappa Statistic". <i>Biometrics</i>. <b>41</b> (3): 795. <a href="/wiki/JSTOR_(identifier)" class="mw-redirect" title="JSTOR (identifier)">JSTOR</a> <a rel="nofollow" class="external text" href="https://www.jstor.org/stable/2531300">2531300</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Biometrics&rft.atitle=Early+History+of+the+Kappa+Statistic&rft.volume=41&rft.issue=3&rft.pages=795&rft.date=1985&rft_id=https%3A%2F%2Fwww.jstor.org%2Fstable%2F2531300%23id-name%3DJSTOR&rft.aulast=Smeeton&rft.aufirst=N.C.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-5">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCohen1960" class="citation journal cs1">Cohen, Jacob (1960). "A coefficient of agreement for nominal scales". <i>Educational and Psychological Measurement</i>. <b>20</b> (1): 37–46. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F001316446002000104">10.1177/001316446002000104</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/1942%2F28116">1942/28116</a></span>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:15926286">15926286</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Educational+and+Psychological+Measurement&rft.atitle=A+coefficient+of+agreement+for+nominal+scales&rft.volume=20&rft.issue=1&rft.pages=37-46&rft.date=1960&rft_id=info%3Ahdl%2F1942%2F28116&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A15926286%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1177%2F001316446002000104&rft.aulast=Cohen&rft.aufirst=Jacob&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-6">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSimWright2005" class="citation journal cs1">Sim, Julius; Wright, Chris C. (2005). <a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fptj%2F85.3.257">"The Kappa Statistic in Reliability Studies: Use, Interpretation, and Sample Size Requirements"</a>. <i>Physical Therapy</i>. <b>85</b> (3): 257–268. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fptj%2F85.3.257">10.1093/ptj/85.3.257</a></span>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1538-6724">1538-6724</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/15733050">15733050</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Therapy&rft.atitle=The+Kappa+Statistic+in+Reliability+Studies%3A+Use%2C+Interpretation%2C+and+Sample+Size+Requirements&rft.volume=85&rft.issue=3&rft.pages=257-268&rft.date=2005&rft.issn=1538-6724&rft_id=info%3Apmid%2F15733050&rft_id=info%3Adoi%2F10.1093%2Fptj%2F85.3.257&rft.aulast=Sim&rft.aufirst=Julius&rft.au=Wright%2C+Chris+C.&rft_id=https%3A%2F%2Fdoi.org%2F10.1093%252Fptj%252F85.3.257&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChicco_D.Warrens_M.J.Jurman_G.2021" class="citation journal cs1">Chicco D.; Warrens M.J.; Jurman G. (June 2021). <a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FACCESS.2021.3084050">"The Matthews correlation coefficient (MCC) is more informative than Cohen's Kappa and Brier score in binary classification assessment"</a>. <i>IEEE Access</i>. <b>9</b>: 78368 - 78381. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2021IEEEA...978368C">2021IEEEA...978368C</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1109%2FACCESS.2021.3084050">10.1109/ACCESS.2021.3084050</a></span>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://hdl.handle.net/10281%2F430460">10281/430460</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Access&rft.atitle=The+Matthews+correlation+coefficient+%28MCC%29+is+more+informative+than+Cohen%27s+Kappa+and+Brier+score+in+binary+classification+assessment&rft.volume=9&rft.pages=78368+-+78381&rft.date=2021-06&rft_id=info%3Ahdl%2F10281%2F430460&rft_id=info%3Adoi%2F10.1109%2FACCESS.2021.3084050&rft_id=info%3Abibcode%2F2021IEEEA...978368C&rft.au=Chicco+D.&rft.au=Warrens+M.J.&rft.au=Jurman+G.&rft_id=https%3A%2F%2Fdoi.org%2F10.1109%252FACCESS.2021.3084050&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHeidke1926" class="citation journal cs1">Heidke, P. (1926-12-01). "Berechnung Des Erfolges Und Der Güte Der Windstärkevorhersagen Im Sturmwarnungsdienst". <i>Geografiska Annaler</i>. <b>8</b> (4): 301–349. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1080%2F20014422.1926.11881138">10.1080/20014422.1926.11881138</a>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2001-4422">2001-4422</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Geografiska+Annaler&rft.atitle=Berechnung+Des+Erfolges+Und+Der+G%C3%BCte+Der+Windst%C3%A4rkevorhersagen+Im+Sturmwarnungsdienst&rft.volume=8&rft.issue=4&rft.pages=301-349&rft.date=1926-12-01&rft_id=info%3Adoi%2F10.1080%2F20014422.1926.11881138&rft.issn=2001-4422&rft.aulast=Heidke&rft.aufirst=P.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPhilosophical_Society_of_Washington_(Washington,_D.C.)1887" class="citation book cs1">Philosophical Society of Washington (Washington, D.C.) (1887). <a rel="nofollow" class="external text" href="https://www.biodiversitylibrary.org/page/55377146"><i>Bulletin of the Philosophical Society of Washington</i></a>. Vol. 10. Washington, D.C.: Published by the co-operation of the Smithsonian Institution. p. 83.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Bulletin+of+the+Philosophical+Society+of+Washington.&rft.place=Washington%2C+D.C.&rft.pages=83&rft.pub=Published+by+the+co-operation+of+the+Smithsonian+Institution&rft.date=1887&rft.au=Philosophical+Society+of+Washington+%28Washington%2C+D.C.%29&rft_id=https%3A%2F%2Fwww.biodiversitylibrary.org%2Fpage%2F55377146&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-Gwet2002-10"><span class="mw-cite-backlink"><b><a href="#cite_ref-Gwet2002_10-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKilem_Gwet2002" class="citation journal cs1">Kilem Gwet (May 2002). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20110707094014/http://agreestat.com/research_papers/inter_rater_reliability_dependency.pdf">"Inter-Rater Reliability: Dependency on Trait Prevalence and Marginal Homogeneity"</a> <span class="cs1-format">(PDF)</span>. <i>Statistical Methods for Inter-Rater Reliability Assessment</i>. <b>2</b>: 1–10. Archived from <a rel="nofollow" class="external text" href="http://agreestat.com/research_papers/inter_rater_reliability_dependency.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 2011-07-07<span class="reference-accessdate">. Retrieved <span class="nowrap">2011-02-02</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Statistical+Methods+for+Inter-Rater+Reliability+Assessment&rft.atitle=Inter-Rater+Reliability%3A+Dependency+on+Trait+Prevalence+and+Marginal+Homogeneity&rft.volume=2&rft.pages=1-10&rft.date=2002-05&rft.au=Kilem+Gwet&rft_id=http%3A%2F%2Fagreestat.com%2Fresearch_papers%2Finter_rater_reliability_dependency.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-BakemanGottman1997-11"><span class="mw-cite-backlink">^ <a href="#cite_ref-BakemanGottman1997_11-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-BakemanGottman1997_11-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBakemanGottman,_J.M.1997" class="citation book cs1">Bakeman, R.; Gottman, J.M. (1997). <i>Observing interaction: An introduction to sequential analysis</i> (2nd ed.). Cambridge, UK: Cambridge University Press. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-521-27593-4" title="Special:BookSources/978-0-521-27593-4"><bdi>978-0-521-27593-4</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Observing+interaction%3A+An+introduction+to+sequential+analysis&rft.place=Cambridge%2C+UK&rft.edition=2nd&rft.pub=Cambridge+University+Press&rft.date=1997&rft.isbn=978-0-521-27593-4&rft.aulast=Bakeman&rft.aufirst=R.&rft.au=Gottman%2C+J.M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-FleissCohenEv1969-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-FleissCohenEv1969_12-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFFleissCohen,_J.Everitt,_B.S.1969" class="citation journal cs1">Fleiss, J.L.; Cohen, J.; Everitt, B.S. (1969). "Large sample standard errors of kappa and weighted kappa". <i>Psychological Bulletin</i>. <b>72</b> (5): 323–327. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1037%2Fh0028106">10.1037/h0028106</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Psychological+Bulletin&rft.atitle=Large+sample+standard+errors+of+kappa+and+weighted+kappa&rft.volume=72&rft.issue=5&rft.pages=323-327&rft.date=1969&rft_id=info%3Adoi%2F10.1037%2Fh0028106&rft.aulast=Fleiss&rft.aufirst=J.L.&rft.au=Cohen%2C+J.&rft.au=Everitt%2C+B.S.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-BakemanRobinson1998-13"><span class="mw-cite-backlink"><b><a href="#cite_ref-BakemanRobinson1998_13-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFRobinsonBakeman,_R.1998" class="citation journal cs1">Robinson, B.F; Bakeman, R. (1998). <a rel="nofollow" class="external text" href="https://doi.org/10.3758%2FBF03209495">"ComKappa: A Windows 95 program for calculating kappa and related statistics"</a>. <i>Behavior Research Methods, Instruments, and Computers</i>. <b>30</b> (4): 731–732. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.3758%2FBF03209495">10.3758/BF03209495</a></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Behavior+Research+Methods%2C+Instruments%2C+and+Computers&rft.atitle=ComKappa%3A+A+Windows+95+program+for+calculating+kappa+and+related+statistics&rft.volume=30&rft.issue=4&rft.pages=731-732&rft.date=1998&rft_id=info%3Adoi%2F10.3758%2FBF03209495&rft.aulast=Robinson&rft.aufirst=B.F&rft.au=Bakeman%2C+R.&rft_id=https%3A%2F%2Fdoi.org%2F10.3758%252FBF03209495&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-SimWright2005-14"><span class="mw-cite-backlink"><b><a href="#cite_ref-SimWright2005_14-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSimWright,_C._C2005" class="citation journal cs1">Sim, J; Wright, C. C (2005). <a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fptj%2F85.3.257">"The Kappa Statistic in Reliability Studies: Use, Interpretation, and Sample Size Requirements"</a>. <i>Physical Therapy</i>. <b>85</b> (3): 257–268. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1093%2Fptj%2F85.3.257">10.1093/ptj/85.3.257</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/15733050">15733050</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Physical+Therapy&rft.atitle=The+Kappa+Statistic+in+Reliability+Studies%3A+Use%2C+Interpretation%2C+and+Sample+Size+Requirements&rft.volume=85&rft.issue=3&rft.pages=257-268&rft.date=2005&rft_id=info%3Adoi%2F10.1093%2Fptj%2F85.3.257&rft_id=info%3Apmid%2F15733050&rft.aulast=Sim&rft.aufirst=J&rft.au=Wright%2C+C.+C&rft_id=https%3A%2F%2Fdoi.org%2F10.1093%252Fptj%252F85.3.257&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-BakemanEtAl1997-15"><span class="mw-cite-backlink"><b><a href="#cite_ref-BakemanEtAl1997_15-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBakemanQuera,_V.McArthur,_D.Robinson,_B._F.1997" class="citation journal cs1">Bakeman, R.; Quera, V.; McArthur, D.; Robinson, B. F. (1997). "Detecting sequential patterns and determining their reliability with fallible observers". <i>Psychological Methods</i>. <b>2</b> (4): 357–370. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1037%2F1082-989X.2.4.357">10.1037/1082-989X.2.4.357</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Psychological+Methods&rft.atitle=Detecting+sequential+patterns+and+determining+their+reliability+with+fallible+observers&rft.volume=2&rft.issue=4&rft.pages=357-370&rft.date=1997&rft_id=info%3Adoi%2F10.1037%2F1082-989X.2.4.357&rft.aulast=Bakeman&rft.aufirst=R.&rft.au=Quera%2C+V.&rft.au=McArthur%2C+D.&rft.au=Robinson%2C+B.+F.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-LandisKoch1977-16"><span class="mw-cite-backlink"><b><a href="#cite_ref-LandisKoch1977_16-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLandisKoch,_G.G.1977" class="citation journal cs1">Landis, J.R.; Koch, G.G. (1977). <a rel="nofollow" class="external text" href="https://repositorio.unal.edu.co/handle/unal/80119">"The measurement of observer agreement for categorical data"</a>. <i>Biometrics</i>. <b>33</b> (1): 159–174. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.2307%2F2529310">10.2307/2529310</a>. <a href="/wiki/JSTOR_(identifier)" class="mw-redirect" title="JSTOR (identifier)">JSTOR</a> <a rel="nofollow" class="external text" href="https://www.jstor.org/stable/2529310">2529310</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/843571">843571</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:11077516">11077516</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Biometrics&rft.atitle=The+measurement+of+observer+agreement+for+categorical+data&rft.volume=33&rft.issue=1&rft.pages=159-174&rft.date=1977&rft_id=info%3Apmid%2F843571&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A11077516%23id-name%3DS2CID&rft_id=https%3A%2F%2Fwww.jstor.org%2Fstable%2F2529310%23id-name%3DJSTOR&rft_id=info%3Adoi%2F10.2307%2F2529310&rft.aulast=Landis&rft.aufirst=J.R.&rft.au=Koch%2C+G.G.&rft_id=https%3A%2F%2Frepositorio.unal.edu.co%2Fhandle%2Funal%2F80119&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-17"><span class="mw-cite-backlink"><b><a href="#cite_ref-17">^</a></b></span> <span class="reference-text">Gwet, K. (2010). "<a rel="nofollow" class="external text" href="http://www.agreestat.com/">Handbook of Inter-Rater Reliability (Second Edition)</a>" <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-9708062-2-2" title="Special:BookSources/978-0-9708062-2-2">978-0-9708062-2-2</a> <sup class="noprint Inline-Template" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citing_sources" title="Wikipedia:Citing sources"><span title="This citation requires a reference to the specific page or range of pages in which the material appears. (April 2012)">page needed</span></a></i>]</sup></span> </li> <li id="cite_note-Fleiss1981-18"><span class="mw-cite-backlink"><b><a href="#cite_ref-Fleiss1981_18-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFFleiss1981" class="citation book cs1">Fleiss, J.L. (1981). <i>Statistical methods for rates and proportions</i> (2nd ed.). New York: John Wiley. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-0-471-26370-8" title="Special:BookSources/978-0-471-26370-8"><bdi>978-0-471-26370-8</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Statistical+methods+for+rates+and+proportions&rft.place=New+York&rft.edition=2nd&rft.pub=John+Wiley&rft.date=1981&rft.isbn=978-0-471-26370-8&rft.aulast=Fleiss&rft.aufirst=J.L.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-Umesh989-19"><span class="mw-cite-backlink"><b><a href="#cite_ref-Umesh989_19-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFUmeshPeterson,_R.A.Sauber_M._H.1989" class="citation journal cs1">Umesh, U. N.; Peterson, R.A.; Sauber M. H. (1989). "Interjudge agreement and the maximum value of kappa". <i>Educational and Psychological Measurement</i>. <b>49</b> (4): 835–850. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1177%2F001316448904900407">10.1177/001316448904900407</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:123306239">123306239</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Educational+and+Psychological+Measurement&rft.atitle=Interjudge+agreement+and+the+maximum+value+of+kappa.&rft.volume=49&rft.issue=4&rft.pages=835-850&rft.date=1989&rft_id=info%3Adoi%2F10.1177%2F001316448904900407&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A123306239%23id-name%3DS2CID&rft.aulast=Umesh&rft.aufirst=U.+N.&rft.au=Peterson%2C+R.A.&rft.au=Sauber+M.+H.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-20"><span class="mw-cite-backlink"><b><a href="#cite_ref-20">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFVieraGarrett2005" class="citation journal cs1">Viera, Anthony J.; Garrett, Joanne M. (2005). "Understanding interobserver agreement: the kappa statistic". <i>Family Medicine</i>. <b>37</b> (5): 360–363. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/15883903">15883903</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Family+Medicine&rft.atitle=Understanding+interobserver+agreement%3A+the+kappa+statistic&rft.volume=37&rft.issue=5&rft.pages=360-363&rft.date=2005&rft_id=info%3Apmid%2F15883903&rft.aulast=Viera&rft.aufirst=Anthony+J.&rft.au=Garrett%2C+Joanne+M.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-SMPJ-21"><span class="mw-cite-backlink"><b><a href="#cite_ref-SMPJ_21-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFStrijbosMartensPrinsJochems2006" class="citation journal cs1">Strijbos, J.; Martens, R.; Prins, F.; Jochems, W. (2006). "Content analysis: What are they talking about?". <i>Computers & Education</i>. <b>46</b>: 29–48. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.397.5780">10.1.1.397.5780</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.compedu.2005.04.002">10.1016/j.compedu.2005.04.002</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:14183447">14183447</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Computers+%26+Education&rft.atitle=Content+analysis%3A+What+are+they+talking+about%3F&rft.volume=46&rft.pages=29-48&rft.date=2006&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.397.5780%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A14183447%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1016%2Fj.compedu.2005.04.002&rft.aulast=Strijbos&rft.aufirst=J.&rft.au=Martens%2C+R.&rft.au=Prins%2C+F.&rft.au=Jochems%2C+W.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-22"><span class="mw-cite-backlink"><b><a href="#cite_ref-22">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFUebersax1987" class="citation journal cs1">Uebersax, JS. (1987). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20160303171301/http://www.na-mic.org/Wiki/images/d/df/Kapp_and_decision_making_models.pdf">"Diversity of decision-making models and the measurement of interrater agreement"</a> <span class="cs1-format">(PDF)</span>. <i>Psychological Bulletin</i>. <b>101</b>: 140–146. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.498.4965">10.1.1.498.4965</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1037%2F0033-2909.101.1.140">10.1037/0033-2909.101.1.140</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:39240770">39240770</a>. Archived from <a rel="nofollow" class="external text" href="http://www.na-mic.org/Wiki/images/d/df/Kapp_and_decision_making_models.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 2016-03-03<span class="reference-accessdate">. Retrieved <span class="nowrap">2010-10-16</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Psychological+Bulletin&rft.atitle=Diversity+of+decision-making+models+and+the+measurement+of+interrater+agreement&rft.volume=101&rft.pages=140-146&rft.date=1987&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.498.4965%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A39240770%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1037%2F0033-2909.101.1.140&rft.aulast=Uebersax&rft.aufirst=JS.&rft_id=http%3A%2F%2Fwww.na-mic.org%2FWiki%2Fimages%2Fd%2Fdf%2FKapp_and_decision_making_models.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-23"><span class="mw-cite-backlink"><b><a href="#cite_ref-23">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDelgadoTibau2019" class="citation journal cs1">Delgado, Rosario; Tibau, Xavier-Andoni (2019-09-26). <a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6762152">"Why Cohen's Kappa should be avoided as performance measure in classification"</a>. <i>PLOS ONE</i>. <b>14</b> (9): e0222916. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2019PLoSO..1422916D">2019PLoSO..1422916D</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1371%2Fjournal.pone.0222916">10.1371/journal.pone.0222916</a></span>. <a href="/wiki/ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1932-6203">1932-6203</a>. <a href="/wiki/PMC_(identifier)" class="mw-redirect" title="PMC (identifier)">PMC</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6762152">6762152</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/31557204">31557204</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=PLOS+ONE&rft.atitle=Why+Cohen%27s+Kappa+should+be+avoided+as+performance+measure+in+classification&rft.volume=14&rft.issue=9&rft.pages=e0222916&rft.date=2019-09-26&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6762152%23id-name%3DPMC&rft_id=info%3Abibcode%2F2019PLoSO..1422916D&rft_id=info%3Apmid%2F31557204&rft_id=info%3Adoi%2F10.1371%2Fjournal.pone.0222916&rft.issn=1932-6203&rft.aulast=Delgado&rft.aufirst=Rosario&rft.au=Tibau%2C+Xavier-Andoni&rft_id=https%3A%2F%2Fwww.ncbi.nlm.nih.gov%2Fpmc%2Farticles%2FPMC6762152&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-Powers2012-24"><span class="mw-cite-backlink"><b><a href="#cite_ref-Powers2012_24-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPowers2012" class="citation conference cs1">Powers, David M. W. (2012). <a rel="nofollow" class="external text" href="http://arquivo.pt/wayback/20160518183306/http://dl.dropbox.com/u/27743223/201209-eacl2012-Kappa.pdf">"The Problem with Kappa"</a> <span class="cs1-format">(PDF)</span>. <i>Conference of the European Chapter of the Association for Computational Linguistics (EACL2012) Joint ROBUS-UNSUP Workshop</i>. Archived from <a rel="nofollow" class="external text" href="https://dl.dropbox.com/u/27743223/201209-eacl2012-Kappa.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 2016-05-18<span class="reference-accessdate">. Retrieved <span class="nowrap">2012-07-20</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=The+Problem+with+Kappa&rft.btitle=Conference+of+the+European+Chapter+of+the+Association+for+Computational+Linguistics+%28EACL2012%29+Joint+ROBUS-UNSUP+Workshop&rft.date=2012&rft.aulast=Powers&rft.aufirst=David+M.+W.&rft_id=http%3A%2F%2Fdl.dropbox.com%2Fu%2F27743223%2F201209-eacl2012-Kappa.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> <li id="cite_note-Cohen1968-25"><span class="mw-cite-backlink"><b><a href="#cite_ref-Cohen1968_25-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCohen1968" class="citation journal cs1">Cohen, J. (1968). "Weighed kappa: Nominal scale agreement with provision for scaled disagreement or partial credit". <i>Psychological Bulletin</i>. <b>70</b> (4): 213–220. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1037%2Fh0026256">10.1037/h0026256</a>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/19673146">19673146</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Psychological+Bulletin&rft.atitle=Weighed+kappa%3A+Nominal+scale+agreement+with+provision+for+scaled+disagreement+or+partial+credit&rft.volume=70&rft.issue=4&rft.pages=213-220&rft.date=1968&rft_id=info%3Adoi%2F10.1037%2Fh0026256&rft_id=info%3Apmid%2F19673146&rft.aulast=Cohen&rft.aufirst=J.&rfr_id=info%3Asid%2Fen.wikipedia.org%3ACohen%27s+kappa" class="Z3988"></span></span> </li> </ol></div> <div class="navbox-styles"><style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="Statistics" style="padding:3px"><table class="nowraplinks hlist mw-collapsible uncollapsed navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Statistics" title="Template:Statistics"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Statistics" title="Template talk:Statistics"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Statistics" title="Special:EditPage/Template:Statistics"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Statistics" style="font-size:114%;margin:0 4em"><a href="/wiki/Statistics" title="Statistics">Statistics</a></div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><a href="/wiki/Outline_of_statistics" title="Outline of statistics">Outline</a></li> <li><a href="/wiki/List_of_statistics_articles" title="List of statistics articles">Index</a></li></ul> </div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Descriptive_statistics" style="font-size:114%;margin:0 4em"><a href="/wiki/Descriptive_statistics" title="Descriptive statistics">Descriptive statistics</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Continuous_probability_distribution" class="mw-redirect" title="Continuous probability distribution">Continuous data</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Central_tendency" title="Central tendency">Center</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Mean" title="Mean">Mean</a> <ul><li><a href="/wiki/Arithmetic_mean" title="Arithmetic mean">Arithmetic</a></li> <li><a href="/wiki/Arithmetic%E2%80%93geometric_mean" title="Arithmetic–geometric mean">Arithmetic-Geometric</a></li> <li><a href="/wiki/Contraharmonic_mean" title="Contraharmonic mean">Contraharmonic</a></li> <li><a href="/wiki/Cubic_mean" title="Cubic mean">Cubic</a></li> <li><a href="/wiki/Generalized_mean" title="Generalized mean">Generalized/power</a></li> <li><a href="/wiki/Geometric_mean" title="Geometric mean">Geometric</a></li> <li><a href="/wiki/Harmonic_mean" title="Harmonic mean">Harmonic</a></li> <li><a href="/wiki/Heronian_mean" title="Heronian mean">Heronian</a></li> <li><a href="/wiki/Heinz_mean" title="Heinz mean">Heinz</a></li> <li><a href="/wiki/Lehmer_mean" title="Lehmer mean">Lehmer</a></li></ul></li> <li><a href="/wiki/Median" title="Median">Median</a></li> <li><a href="/wiki/Mode_(statistics)" title="Mode (statistics)">Mode</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Statistical_dispersion" title="Statistical dispersion">Dispersion</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Average_absolute_deviation" title="Average absolute deviation">Average absolute deviation</a></li> <li><a href="/wiki/Coefficient_of_variation" title="Coefficient of variation">Coefficient of variation</a></li> <li><a href="/wiki/Interquartile_range" title="Interquartile range">Interquartile range</a></li> <li><a href="/wiki/Percentile" title="Percentile">Percentile</a></li> <li><a href="/wiki/Range_(statistics)" title="Range (statistics)">Range</a></li> <li><a href="/wiki/Standard_deviation" title="Standard deviation">Standard deviation</a></li> <li><a href="/wiki/Variance#Sample_variance" title="Variance">Variance</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Shape_of_the_distribution" class="mw-redirect" title="Shape of the distribution">Shape</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Central_limit_theorem" title="Central limit theorem">Central limit theorem</a></li> <li><a href="/wiki/Moment_(mathematics)" title="Moment (mathematics)">Moments</a> <ul><li><a href="/wiki/Kurtosis" title="Kurtosis">Kurtosis</a></li> <li><a href="/wiki/L-moment" title="L-moment">L-moments</a></li> <li><a href="/wiki/Skewness" title="Skewness">Skewness</a></li></ul></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Count_data" title="Count data">Count data</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Index_of_dispersion" title="Index of dispersion">Index of dispersion</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em">Summary tables</th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Contingency_table" title="Contingency table">Contingency table</a></li> <li><a href="/wiki/Frequency_distribution" class="mw-redirect" title="Frequency distribution">Frequency distribution</a></li> <li><a href="/wiki/Grouped_data" title="Grouped data">Grouped data</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">Dependence</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Partial_correlation" title="Partial correlation">Partial correlation</a></li> <li><a href="/wiki/Pearson_correlation_coefficient" title="Pearson correlation coefficient">Pearson product-moment correlation</a></li> <li><a href="/wiki/Rank_correlation" title="Rank correlation">Rank correlation</a> <ul><li><a href="/wiki/Kendall_rank_correlation_coefficient" title="Kendall rank correlation coefficient">Kendall's τ</a></li> <li><a href="/wiki/Spearman%27s_rank_correlation_coefficient" title="Spearman's rank correlation coefficient">Spearman's ρ</a></li></ul></li> <li><a href="/wiki/Scatter_plot" title="Scatter plot">Scatter plot</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Statistical_graphics" title="Statistical graphics">Graphics</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Bar_chart" title="Bar chart">Bar chart</a></li> <li><a href="/wiki/Biplot" title="Biplot">Biplot</a></li> <li><a href="/wiki/Box_plot" title="Box plot">Box plot</a></li> <li><a href="/wiki/Control_chart" title="Control chart">Control chart</a></li> <li><a href="/wiki/Correlogram" title="Correlogram">Correlogram</a></li> <li><a href="/wiki/Fan_chart_(statistics)" title="Fan chart (statistics)">Fan chart</a></li> <li><a href="/wiki/Forest_plot" title="Forest plot">Forest plot</a></li> <li><a href="/wiki/Histogram" title="Histogram">Histogram</a></li> <li><a href="/wiki/Pie_chart" title="Pie chart">Pie chart</a></li> <li><a href="/wiki/Q%E2%80%93Q_plot" title="Q–Q plot">Q–Q plot</a></li> <li><a href="/wiki/Radar_chart" title="Radar chart">Radar chart</a></li> <li><a href="/wiki/Run_chart" title="Run chart">Run chart</a></li> <li><a href="/wiki/Scatter_plot" title="Scatter plot">Scatter plot</a></li> <li><a href="/wiki/Stem-and-leaf_display" title="Stem-and-leaf display">Stem-and-leaf display</a></li> <li><a href="/wiki/Violin_plot" title="Violin plot">Violin plot</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Data_collection" style="font-size:114%;margin:0 4em"><a href="/wiki/Data_collection" title="Data collection">Data collection</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Design_of_experiments" title="Design of experiments">Study design</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Effect_size" title="Effect size">Effect size</a></li> <li><a href="/wiki/Missing_data" title="Missing data">Missing data</a></li> <li><a href="/wiki/Optimal_design" class="mw-redirect" title="Optimal design">Optimal design</a></li> <li><a href="/wiki/Statistical_population" title="Statistical population">Population</a></li> <li><a href="/wiki/Replication_(statistics)" title="Replication (statistics)">Replication</a></li> <li><a href="/wiki/Sample_size_determination" title="Sample size determination">Sample size determination</a></li> <li><a href="/wiki/Statistic" title="Statistic">Statistic</a></li> <li><a href="/wiki/Statistical_power" class="mw-redirect" title="Statistical power">Statistical power</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Survey_methodology" title="Survey methodology">Survey methodology</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Sampling_(statistics)" title="Sampling (statistics)">Sampling</a> <ul><li><a href="/wiki/Cluster_sampling" title="Cluster sampling">Cluster</a></li> <li><a href="/wiki/Stratified_sampling" title="Stratified sampling">Stratified</a></li></ul></li> <li><a href="/wiki/Opinion_poll" title="Opinion poll">Opinion poll</a></li> <li><a href="/wiki/Questionnaire" title="Questionnaire">Questionnaire</a></li> <li><a href="/wiki/Standard_error" title="Standard error">Standard error</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Experiment" title="Experiment">Controlled experiments</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Blocking_(statistics)" title="Blocking (statistics)">Blocking</a></li> <li><a href="/wiki/Factorial_experiment" title="Factorial experiment">Factorial experiment</a></li> <li><a href="/wiki/Interaction_(statistics)" title="Interaction (statistics)">Interaction</a></li> <li><a href="/wiki/Random_assignment" title="Random assignment">Random assignment</a></li> <li><a href="/wiki/Randomized_controlled_trial" title="Randomized controlled trial">Randomized controlled trial</a></li> <li><a href="/wiki/Randomized_experiment" title="Randomized experiment">Randomized experiment</a></li> <li><a href="/wiki/Scientific_control" title="Scientific control">Scientific control</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em">Adaptive designs</th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Adaptive_clinical_trial" class="mw-redirect" title="Adaptive clinical trial">Adaptive clinical trial</a></li> <li><a href="/wiki/Stochastic_approximation" title="Stochastic approximation">Stochastic approximation</a></li> <li><a href="/wiki/Up-and-Down_Designs" class="mw-redirect" title="Up-and-Down Designs">Up-and-down designs</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Observational_study" title="Observational study">Observational studies</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cohort_study" title="Cohort study">Cohort study</a></li> <li><a href="/wiki/Cross-sectional_study" title="Cross-sectional study">Cross-sectional study</a></li> <li><a href="/wiki/Natural_experiment" title="Natural experiment">Natural experiment</a></li> <li><a href="/wiki/Quasi-experiment" title="Quasi-experiment">Quasi-experiment</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Statistical_inference" style="font-size:114%;margin:0 4em"><a href="/wiki/Statistical_inference" title="Statistical inference">Statistical inference</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Statistical_theory" title="Statistical theory">Statistical theory</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Population_(statistics)" class="mw-redirect" title="Population (statistics)">Population</a></li> <li><a href="/wiki/Statistic" title="Statistic">Statistic</a></li> <li><a href="/wiki/Probability_distribution" title="Probability distribution">Probability distribution</a></li> <li><a href="/wiki/Sampling_distribution" title="Sampling distribution">Sampling distribution</a> <ul><li><a href="/wiki/Order_statistic" title="Order statistic">Order statistic</a></li></ul></li> <li><a href="/wiki/Empirical_distribution_function" title="Empirical distribution function">Empirical distribution</a> <ul><li><a href="/wiki/Density_estimation" title="Density estimation">Density estimation</a></li></ul></li> <li><a href="/wiki/Statistical_model" title="Statistical model">Statistical model</a> <ul><li><a href="/wiki/Model_specification" class="mw-redirect" title="Model specification">Model specification</a></li> <li><a href="/wiki/Lp_space" title="Lp space">L<sup><i>p</i></sup> space</a></li></ul></li> <li><a href="/wiki/Statistical_parameter" title="Statistical parameter">Parameter</a> <ul><li><a href="/wiki/Location_parameter" title="Location parameter">location</a></li> <li><a href="/wiki/Scale_parameter" title="Scale parameter">scale</a></li> <li><a href="/wiki/Shape_parameter" title="Shape parameter">shape</a></li></ul></li> <li><a href="/wiki/Parametric_statistics" title="Parametric statistics">Parametric family</a> <ul><li><a href="/wiki/Likelihood_function" title="Likelihood function">Likelihood</a> <a href="/wiki/Monotone_likelihood_ratio" title="Monotone likelihood ratio"><span style="font-size:85%;">(monotone)</span></a></li> <li><a href="/wiki/Location%E2%80%93scale_family" title="Location–scale family">Location–scale family</a></li> <li><a href="/wiki/Exponential_family" title="Exponential family">Exponential family</a></li></ul></li> <li><a href="/wiki/Completeness_(statistics)" title="Completeness (statistics)">Completeness</a></li> <li><a href="/wiki/Sufficient_statistic" title="Sufficient statistic">Sufficiency</a></li> <li><a href="/wiki/Plug-in_principle" class="mw-redirect" title="Plug-in principle">Statistical functional</a> <ul><li><a href="/wiki/Bootstrapping_(statistics)" title="Bootstrapping (statistics)">Bootstrap</a></li> <li><a href="/wiki/U-statistic" title="U-statistic">U</a></li> <li><a href="/wiki/V-statistic" title="V-statistic">V</a></li></ul></li> <li><a href="/wiki/Optimal_decision" title="Optimal decision">Optimal decision</a> <ul><li><a href="/wiki/Loss_function" title="Loss function">loss function</a></li></ul></li> <li><a href="/wiki/Efficiency_(statistics)" title="Efficiency (statistics)">Efficiency</a></li> <li><a href="/wiki/Statistical_distance" title="Statistical distance">Statistical distance</a> <ul><li><a href="/wiki/Divergence_(statistics)" title="Divergence (statistics)">divergence</a></li></ul></li> <li><a href="/wiki/Asymptotic_theory_(statistics)" title="Asymptotic theory (statistics)">Asymptotics</a></li> <li><a href="/wiki/Robust_statistics" title="Robust statistics">Robustness</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Frequentist_inference" title="Frequentist inference">Frequentist inference</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Point_estimation" title="Point estimation">Point estimation</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Estimating_equations" title="Estimating equations">Estimating equations</a> <ul><li><a href="/wiki/Maximum_likelihood" class="mw-redirect" title="Maximum likelihood">Maximum likelihood</a></li> <li><a href="/wiki/Method_of_moments_(statistics)" title="Method of moments (statistics)">Method of moments</a></li> <li><a href="/wiki/M-estimator" title="M-estimator">M-estimator</a></li> <li><a href="/wiki/Minimum_distance_estimation" class="mw-redirect" title="Minimum distance estimation">Minimum distance</a></li></ul></li> <li><a href="/wiki/Bias_of_an_estimator" title="Bias of an estimator">Unbiased estimators</a> <ul><li><a href="/wiki/Minimum-variance_unbiased_estimator" title="Minimum-variance unbiased estimator">Mean-unbiased minimum-variance</a> <ul><li><a href="/wiki/Rao%E2%80%93Blackwell_theorem" title="Rao–Blackwell theorem">Rao–Blackwellization</a></li> <li><a href="/wiki/Lehmann%E2%80%93Scheff%C3%A9_theorem" title="Lehmann–Scheffé theorem">Lehmann–Scheffé theorem</a></li></ul></li> <li><a href="/wiki/Median-unbiased_estimator" class="mw-redirect" title="Median-unbiased estimator">Median unbiased</a></li></ul></li> <li><a href="/wiki/Plug-in_principle" class="mw-redirect" title="Plug-in principle">Plug-in</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Interval_estimation" title="Interval estimation">Interval estimation</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Confidence_interval" title="Confidence interval">Confidence interval</a></li> <li><a href="/wiki/Pivotal_quantity" title="Pivotal quantity">Pivot</a></li> <li><a href="/wiki/Likelihood_interval" class="mw-redirect" title="Likelihood interval">Likelihood interval</a></li> <li><a href="/wiki/Prediction_interval" title="Prediction interval">Prediction interval</a></li> <li><a href="/wiki/Tolerance_interval" title="Tolerance interval">Tolerance interval</a></li> <li><a href="/wiki/Resampling_(statistics)" title="Resampling (statistics)">Resampling</a> <ul><li><a href="/wiki/Bootstrapping_(statistics)" title="Bootstrapping (statistics)">Bootstrap</a></li> <li><a href="/wiki/Jackknife_resampling" title="Jackknife resampling">Jackknife</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Statistical_hypothesis_testing" class="mw-redirect" title="Statistical hypothesis testing">Testing hypotheses</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/One-_and_two-tailed_tests" title="One- and two-tailed tests">1- & 2-tails</a></li> <li><a href="/wiki/Power_(statistics)" title="Power (statistics)">Power</a> <ul><li><a href="/wiki/Uniformly_most_powerful_test" title="Uniformly most powerful test">Uniformly most powerful test</a></li></ul></li> <li><a href="/wiki/Permutation_test" title="Permutation test">Permutation test</a> <ul><li><a href="/wiki/Randomization_test" class="mw-redirect" title="Randomization test">Randomization test</a></li></ul></li> <li><a href="/wiki/Multiple_comparisons" class="mw-redirect" title="Multiple comparisons">Multiple comparisons</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Parametric_statistics" title="Parametric statistics">Parametric tests</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Likelihood-ratio_test" title="Likelihood-ratio test">Likelihood-ratio</a></li> <li><a href="/wiki/Score_test" title="Score test">Score/Lagrange multiplier</a></li> <li><a href="/wiki/Wald_test" title="Wald test">Wald</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/List_of_statistical_tests" title="List of statistical tests">Specific tests</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><td colspan="2" class="navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Z-test" title="Z-test"><i>Z</i>-test <span style="font-size:85%;">(normal)</span></a></li> <li><a href="/wiki/Student%27s_t-test" title="Student's t-test">Student's <i>t</i>-test</a></li> <li><a href="/wiki/F-test" title="F-test"><i>F</i>-test</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Goodness_of_fit" title="Goodness of fit">Goodness of fit</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Chi-squared_test" title="Chi-squared test">Chi-squared</a></li> <li><a href="/wiki/G-test" title="G-test"><i>G</i>-test</a></li> <li><a href="/wiki/Kolmogorov%E2%80%93Smirnov_test" title="Kolmogorov–Smirnov test">Kolmogorov–Smirnov</a></li> <li><a href="/wiki/Anderson%E2%80%93Darling_test" title="Anderson–Darling test">Anderson–Darling</a></li> <li><a href="/wiki/Lilliefors_test" title="Lilliefors test">Lilliefors</a></li> <li><a href="/wiki/Jarque%E2%80%93Bera_test" title="Jarque–Bera test">Jarque–Bera</a></li> <li><a href="/wiki/Shapiro%E2%80%93Wilk_test" title="Shapiro–Wilk test">Normality <span style="font-size:85%;">(Shapiro–Wilk)</span></a></li> <li><a href="/wiki/Likelihood-ratio_test" title="Likelihood-ratio test">Likelihood-ratio test</a></li> <li><a href="/wiki/Model_selection" title="Model selection">Model selection</a> <ul><li><a href="/wiki/Cross-validation_(statistics)" title="Cross-validation (statistics)">Cross validation</a></li> <li><a href="/wiki/Akaike_information_criterion" title="Akaike information criterion">AIC</a></li> <li><a href="/wiki/Bayesian_information_criterion" title="Bayesian information criterion">BIC</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Rank_statistics" class="mw-redirect" title="Rank statistics">Rank statistics</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Sign_test" title="Sign test">Sign</a> <ul><li><a href="/wiki/Sample_median" class="mw-redirect" title="Sample median">Sample median</a></li></ul></li> <li><a href="/wiki/Wilcoxon_signed-rank_test" title="Wilcoxon signed-rank test">Signed rank <span style="font-size:85%;">(Wilcoxon)</span></a> <ul><li><a href="/wiki/Hodges%E2%80%93Lehmann_estimator" title="Hodges–Lehmann estimator">Hodges–Lehmann estimator</a></li></ul></li> <li><a href="/wiki/Mann%E2%80%93Whitney_U_test" title="Mann–Whitney U test">Rank sum <span style="font-size:85%;">(Mann–Whitney)</span></a></li> <li><a href="/wiki/Nonparametric_statistics" title="Nonparametric statistics">Nonparametric</a> <a href="/wiki/Analysis_of_variance" title="Analysis of variance">anova</a> <ul><li><a href="/wiki/Kruskal%E2%80%93Wallis_test" title="Kruskal–Wallis test">1-way <span style="font-size:85%;">(Kruskal–Wallis)</span></a></li> <li><a href="/wiki/Friedman_test" title="Friedman test">2-way <span style="font-size:85%;">(Friedman)</span></a></li> <li><a href="/wiki/Jonckheere%27s_trend_test" title="Jonckheere's trend test">Ordered alternative <span style="font-size:85%;">(Jonckheere–Terpstra)</span></a></li></ul></li> <li><a href="/wiki/Van_der_Waerden_test" title="Van der Waerden test">Van der Waerden test</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Bayesian_inference" title="Bayesian inference">Bayesian inference</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Bayesian_probability" title="Bayesian probability">Bayesian probability</a> <ul><li><a href="/wiki/Prior_probability" title="Prior probability">prior</a></li> <li><a href="/wiki/Posterior_probability" title="Posterior probability">posterior</a></li></ul></li> <li><a href="/wiki/Credible_interval" title="Credible interval">Credible interval</a></li> <li><a href="/wiki/Bayes_factor" title="Bayes factor">Bayes factor</a></li> <li><a href="/wiki/Bayes_estimator" title="Bayes estimator">Bayesian estimator</a> <ul><li><a href="/wiki/Maximum_a_posteriori_estimation" title="Maximum a posteriori estimation">Maximum posterior estimator</a></li></ul></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="CorrelationRegression_analysis" style="font-size:114%;margin:0 4em"><div class="hlist"><ul><li><a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">Correlation</a></li><li><a href="/wiki/Regression_analysis" title="Regression analysis">Regression analysis</a></li></ul></div></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Correlation_and_dependence" class="mw-redirect" title="Correlation and dependence">Correlation</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Pearson_product-moment_correlation_coefficient" class="mw-redirect" title="Pearson product-moment correlation coefficient">Pearson product-moment</a></li> <li><a href="/wiki/Partial_correlation" title="Partial correlation">Partial correlation</a></li> <li><a href="/wiki/Confounding" title="Confounding">Confounding variable</a></li> <li><a href="/wiki/Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Regression_analysis" title="Regression analysis">Regression analysis</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Errors_and_residuals" title="Errors and residuals">Errors and residuals</a></li> <li><a href="/wiki/Regression_validation" title="Regression validation">Regression validation</a></li> <li><a href="/wiki/Mixed_model" title="Mixed model">Mixed effects models</a></li> <li><a href="/wiki/Simultaneous_equations_model" title="Simultaneous equations model">Simultaneous equations models</a></li> <li><a href="/wiki/Multivariate_adaptive_regression_splines" class="mw-redirect" title="Multivariate adaptive regression splines">Multivariate adaptive regression splines (MARS)</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Linear_regression" title="Linear regression">Linear regression</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Simple_linear_regression" title="Simple linear regression">Simple linear regression</a></li> <li><a href="/wiki/Ordinary_least_squares" title="Ordinary least squares">Ordinary least squares</a></li> <li><a href="/wiki/General_linear_model" title="General linear model">General linear model</a></li> <li><a href="/wiki/Bayesian_linear_regression" title="Bayesian linear regression">Bayesian regression</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em">Non-standard predictors</th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Nonlinear_regression" title="Nonlinear regression">Nonlinear regression</a></li> <li><a href="/wiki/Nonparametric_regression" title="Nonparametric regression">Nonparametric</a></li> <li><a href="/wiki/Semiparametric_regression" title="Semiparametric regression">Semiparametric</a></li> <li><a href="/wiki/Isotonic_regression" title="Isotonic regression">Isotonic</a></li> <li><a href="/wiki/Robust_regression" title="Robust regression">Robust</a></li> <li><a href="/wiki/Heteroscedasticity" class="mw-redirect" title="Heteroscedasticity">Heteroscedasticity</a></li> <li><a href="/wiki/Homoscedasticity" class="mw-redirect" title="Homoscedasticity">Homoscedasticity</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Generalized_linear_model" title="Generalized linear model">Generalized linear model</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Exponential_family" title="Exponential family">Exponential families</a></li> <li><a href="/wiki/Logistic_regression" title="Logistic regression">Logistic <span style="font-size:85%;">(Bernoulli)</span></a> / <a href="/wiki/Binomial_regression" title="Binomial regression">Binomial</a> / <a href="/wiki/Poisson_regression" title="Poisson regression">Poisson regressions</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Partition_of_sums_of_squares" title="Partition of sums of squares">Partition of variance</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Analysis_of_variance" title="Analysis of variance">Analysis of variance (ANOVA, anova)</a></li> <li><a href="/wiki/Analysis_of_covariance" title="Analysis of covariance">Analysis of covariance</a></li> <li><a href="/wiki/Multivariate_analysis_of_variance" title="Multivariate analysis of variance">Multivariate ANOVA</a></li> <li><a href="/wiki/Degrees_of_freedom_(statistics)" title="Degrees of freedom (statistics)">Degrees of freedom</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible uncollapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Categorical_/_Multivariate_/_Time-series_/_Survival_analysis" style="font-size:114%;margin:0 4em"><a href="/wiki/Categorical_variable" title="Categorical variable">Categorical</a> / <a href="/wiki/Multivariate_statistics" title="Multivariate statistics">Multivariate</a> / <a href="/wiki/Time_series" title="Time series">Time-series</a> / <a href="/wiki/Survival_analysis" title="Survival analysis">Survival analysis</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Categorical_variable" title="Categorical variable">Categorical</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a class="mw-selflink selflink">Cohen's kappa</a></li> <li><a href="/wiki/Contingency_table" title="Contingency table">Contingency table</a></li> <li><a href="/wiki/Graphical_model" title="Graphical model">Graphical model</a></li> <li><a href="/wiki/Poisson_regression" title="Poisson regression">Log-linear model</a></li> <li><a href="/wiki/McNemar%27s_test" title="McNemar's test">McNemar's test</a></li> <li><a href="/wiki/Cochran%E2%80%93Mantel%E2%80%93Haenszel_statistics" title="Cochran–Mantel–Haenszel statistics">Cochran–Mantel–Haenszel statistics</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Multivariate_statistics" title="Multivariate statistics">Multivariate</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/General_linear_model" title="General linear model">Regression</a></li> <li><a href="/wiki/Multivariate_analysis_of_variance" title="Multivariate analysis of variance">Manova</a></li> <li><a href="/wiki/Principal_component_analysis" title="Principal component analysis">Principal components</a></li> <li><a href="/wiki/Canonical_correlation" title="Canonical correlation">Canonical correlation</a></li> <li><a href="/wiki/Linear_discriminant_analysis" title="Linear discriminant analysis">Discriminant analysis</a></li> <li><a href="/wiki/Cluster_analysis" title="Cluster analysis">Cluster analysis</a></li> <li><a href="/wiki/Statistical_classification" title="Statistical classification">Classification</a></li> <li><a href="/wiki/Structural_equation_modeling" title="Structural equation modeling">Structural equation model</a> <ul><li><a href="/wiki/Factor_analysis" title="Factor analysis">Factor analysis</a></li></ul></li> <li><a href="/wiki/Multivariate_distribution" class="mw-redirect" title="Multivariate distribution">Multivariate distributions</a> <ul><li><a href="/wiki/Elliptical_distribution" title="Elliptical distribution">Elliptical distributions</a> <ul><li><a href="/wiki/Multivariate_normal_distribution" title="Multivariate normal distribution">Normal</a></li></ul></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Time_series" title="Time series">Time-series</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;">General</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Decomposition_of_time_series" title="Decomposition of time series">Decomposition</a></li> <li><a href="/wiki/Trend_estimation" class="mw-redirect" title="Trend estimation">Trend</a></li> <li><a href="/wiki/Stationary_process" title="Stationary process">Stationarity</a></li> <li><a href="/wiki/Seasonal_adjustment" title="Seasonal adjustment">Seasonal adjustment</a></li> <li><a href="/wiki/Exponential_smoothing" title="Exponential smoothing">Exponential smoothing</a></li> <li><a href="/wiki/Cointegration" title="Cointegration">Cointegration</a></li> <li><a href="/wiki/Structural_break" title="Structural break">Structural break</a></li> <li><a href="/wiki/Granger_causality" title="Granger causality">Granger causality</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;">Specific tests</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Dickey%E2%80%93Fuller_test" title="Dickey–Fuller test">Dickey–Fuller</a></li> <li><a href="/wiki/Johansen_test" title="Johansen test">Johansen</a></li> <li><a href="/wiki/Ljung%E2%80%93Box_test" title="Ljung–Box test">Q-statistic <span style="font-size:85%;">(Ljung–Box)</span></a></li> <li><a href="/wiki/Durbin%E2%80%93Watson_statistic" title="Durbin–Watson statistic">Durbin–Watson</a></li> <li><a href="/wiki/Breusch%E2%80%93Godfrey_test" title="Breusch–Godfrey test">Breusch–Godfrey</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Time_domain" title="Time domain">Time domain</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Autocorrelation" title="Autocorrelation">Autocorrelation (ACF)</a> <ul><li><a href="/wiki/Partial_autocorrelation_function" title="Partial autocorrelation function">partial (PACF)</a></li></ul></li> <li><a href="/wiki/Cross-correlation" title="Cross-correlation">Cross-correlation (XCF)</a></li> <li><a href="/wiki/Autoregressive%E2%80%93moving-average_model" class="mw-redirect" title="Autoregressive–moving-average model">ARMA model</a></li> <li><a href="/wiki/Box%E2%80%93Jenkins_method" title="Box–Jenkins method">ARIMA model <span style="font-size:85%;">(Box–Jenkins)</span></a></li> <li><a href="/wiki/Autoregressive_conditional_heteroskedasticity" title="Autoregressive conditional heteroskedasticity">Autoregressive conditional heteroskedasticity (ARCH)</a></li> <li><a href="/wiki/Vector_autoregression" title="Vector autoregression">Vector autoregression (VAR)</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Frequency_domain" title="Frequency domain">Frequency domain</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Spectral_density_estimation" title="Spectral density estimation">Spectral density estimation</a></li> <li><a href="/wiki/Fourier_analysis" title="Fourier analysis">Fourier analysis</a></li> <li><a href="/wiki/Least-squares_spectral_analysis" title="Least-squares spectral analysis">Least-squares spectral analysis</a></li> <li><a href="/wiki/Wavelet" title="Wavelet">Wavelet</a></li> <li><a href="/wiki/Whittle_likelihood" title="Whittle likelihood">Whittle likelihood</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Survival_analysis" title="Survival analysis">Survival</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Survival_function" title="Survival function">Survival function</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Kaplan%E2%80%93Meier_estimator" title="Kaplan–Meier estimator">Kaplan–Meier estimator (product limit)</a></li> <li><a href="/wiki/Proportional_hazards_model" title="Proportional hazards model">Proportional hazards models</a></li> <li><a href="/wiki/Accelerated_failure_time_model" title="Accelerated failure time model">Accelerated failure time (AFT) model</a></li> <li><a href="/wiki/First-hitting-time_model" title="First-hitting-time model">First hitting time</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;"><a href="/wiki/Failure_rate" title="Failure rate">Hazard function</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Nelson%E2%80%93Aalen_estimator" title="Nelson–Aalen estimator">Nelson–Aalen estimator</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%;font-weight:normal;">Test</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Log-rank_test" class="mw-redirect" title="Log-rank test">Log-rank test</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks mw-collapsible mw-collapsed navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Applications" style="font-size:114%;margin:0 4em"><a href="/wiki/List_of_fields_of_application_of_statistics" title="List of fields of application of statistics">Applications</a></div></th></tr><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Biostatistics" title="Biostatistics">Biostatistics</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Bioinformatics" title="Bioinformatics">Bioinformatics</a></li> <li><a href="/wiki/Clinical_trial" title="Clinical trial">Clinical trials</a> / <a href="/wiki/Clinical_study_design" title="Clinical study design">studies</a></li> <li><a href="/wiki/Epidemiology" title="Epidemiology">Epidemiology</a></li> <li><a href="/wiki/Medical_statistics" title="Medical statistics">Medical statistics</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Engineering_statistics" title="Engineering statistics">Engineering statistics</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Chemometrics" title="Chemometrics">Chemometrics</a></li> <li><a href="/wiki/Methods_engineering" title="Methods engineering">Methods engineering</a></li> <li><a href="/wiki/Probabilistic_design" title="Probabilistic design">Probabilistic design</a></li> <li><a href="/wiki/Statistical_process_control" title="Statistical process control">Process</a> / <a href="/wiki/Quality_control" title="Quality control">quality control</a></li> <li><a href="/wiki/Reliability_engineering" title="Reliability engineering">Reliability</a></li> <li><a href="/wiki/System_identification" title="System identification">System identification</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Social_statistics" title="Social statistics">Social statistics</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Actuarial_science" title="Actuarial science">Actuarial science</a></li> <li><a href="/wiki/Census" title="Census">Census</a></li> <li><a href="/wiki/Crime_statistics" title="Crime statistics">Crime statistics</a></li> <li><a href="/wiki/Demographic_statistics" title="Demographic statistics">Demography</a></li> <li><a href="/wiki/Econometrics" title="Econometrics">Econometrics</a></li> <li><a href="/wiki/Jurimetrics" title="Jurimetrics">Jurimetrics</a></li> <li><a href="/wiki/National_accounts" title="National accounts">National accounts</a></li> <li><a href="/wiki/Official_statistics" title="Official statistics">Official statistics</a></li> <li><a href="/wiki/Population_statistics" class="mw-redirect" title="Population statistics">Population statistics</a></li> <li><a href="/wiki/Psychometrics" title="Psychometrics">Psychometrics</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:12.5em"><a href="/wiki/Spatial_analysis" title="Spatial analysis">Spatial statistics</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cartography" title="Cartography">Cartography</a></li> <li><a href="/wiki/Environmental_statistics" title="Environmental statistics">Environmental statistics</a></li> <li><a href="/wiki/Geographic_information_system" title="Geographic information system">Geographic information system</a></li> <li><a href="/wiki/Geostatistics" title="Geostatistics">Geostatistics</a></li> <li><a href="/wiki/Kriging" title="Kriging">Kriging</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr></tbody></table><div></div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><span class="noviewer" typeof="mw:File"><span title="Category"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/16px-Symbol_category_class.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/23px-Symbol_category_class.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/9/96/Symbol_category_class.svg/31px-Symbol_category_class.svg.png 2x" data-file-width="180" data-file-height="185" /></span></span><b><a href="/wiki/Category:Statistics" title="Category:Statistics">Category</a></b></li> <li><b><span class="nowrap"><span class="noviewer" typeof="mw:File"><a href="/wiki/File:Nuvola_apps_edu_mathematics_blue-p.svg" class="mw-file-description"><img alt="icon" src="//upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Nuvola_apps_edu_mathematics_blue-p.svg/28px-Nuvola_apps_edu_mathematics_blue-p.svg.png" decoding="async" width="28" height="28" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Nuvola_apps_edu_mathematics_blue-p.svg/42px-Nuvola_apps_edu_mathematics_blue-p.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3e/Nuvola_apps_edu_mathematics_blue-p.svg/56px-Nuvola_apps_edu_mathematics_blue-p.svg.png 2x" data-file-width="128" data-file-height="128" /></a></span> </span><a href="/wiki/Portal:Mathematics" title="Portal:Mathematics">Mathematics portal</a></b></li> <li><span class="noviewer" typeof="mw:File"><span title="Commons page"><img alt="" src="//upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/12px-Commons-logo.svg.png" decoding="async" width="12" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/18px-Commons-logo.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/24px-Commons-logo.svg.png 2x" data-file-width="1024" data-file-height="1376" /></span></span><b><a href="https://commons.wikimedia.org/wiki/Category:Statistics" class="extiw" title="commons:Category:Statistics">Commons</a></b></li> <li><span class="noviewer" typeof="mw:File"><span title="WikiProject"><img alt="" src="//upload.wikimedia.org/wikipedia/commons/thumb/3/37/People_icon.svg/16px-People_icon.svg.png" decoding="async" width="16" height="16" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/37/People_icon.svg/24px-People_icon.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/37/People_icon.svg/32px-People_icon.svg.png 2x" data-file-width="100" data-file-height="100" /></span></span> <b><a href="/wiki/Wikipedia:WikiProject_Statistics" title="Wikipedia:WikiProject Statistics">WikiProject</a></b></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235"></div><div role="navigation" class="navbox" aria-labelledby="Machine_learning_evaluation_metrics" style="padding:3px"><table class="nowraplinks mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1239400231"><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Machine_learning_evaluation_metrics" title="Template:Machine learning evaluation metrics"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Machine_learning_evaluation_metrics" title="Template talk:Machine learning evaluation metrics"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Machine_learning_evaluation_metrics" title="Special:EditPage/Template:Machine learning evaluation metrics"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Machine_learning_evaluation_metrics" style="font-size:114%;margin:0 4em"><a href="/wiki/Machine_learning" title="Machine learning">Machine learning</a> evaluation metrics</div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Regression_analysis" title="Regression analysis">Regression</a></th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Mean_squared_error" title="Mean squared error">MSE</a></li> <li><a href="/wiki/Mean_absolute_error" title="Mean absolute error">MAE</a></li> <li><a href="/wiki/Symmetric_mean_absolute_percentage_error" title="Symmetric mean absolute percentage error">sMAPE</a></li> <li><a href="/wiki/Mean_absolute_percentage_error" title="Mean absolute percentage error">MAPE</a></li> <li><a href="/wiki/Mean_absolute_scaled_error" title="Mean absolute scaled error">MASE</a></li> <li><a href="/wiki/Mean_squared_prediction_error" title="Mean squared prediction error">MSPE</a></li> <li><a href="/wiki/Root_mean_square" title="Root mean square">RMS</a></li> <li><a href="/wiki/Root-mean-square_deviation" class="mw-redirect" title="Root-mean-square deviation">RMSE/RMSD</a></li> <li><a href="/wiki/Coefficient_of_determination" title="Coefficient of determination">R<sup>2</sup></a></li> <li><a href="/wiki/Mean_directional_accuracy" title="Mean directional accuracy">MDA</a></li> <li><a href="/wiki/Median_absolute_deviation" title="Median absolute deviation">MAD</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Statistical_classification" title="Statistical classification">Classification</a></th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/F-score" title="F-score">F-score</a></li> <li><a href="/wiki/P4-metric" title="P4-metric">P4</a></li> <li><a href="/wiki/Accuracy_and_precision" title="Accuracy and precision">Accuracy</a></li> <li><a href="/wiki/Precision_and_recall" title="Precision and recall">Precision</a></li> <li><a href="/wiki/Precision_and_recall" title="Precision and recall">Recall</a></li> <li><a class="mw-selflink selflink">Kappa</a></li> <li><a href="/wiki/Phi_coefficient" title="Phi coefficient">MCC</a></li> <li><a href="/wiki/Receiver_operating_characteristic#Area_under_the_curve" title="Receiver operating characteristic">AUC</a></li> <li><a href="/wiki/Receiver_operating_characteristic" title="Receiver operating characteristic">ROC</a></li> <li><a href="/wiki/Sensitivity_and_specificity" title="Sensitivity and specificity">Sensitivity and specificity</a></li> <li><a href="/wiki/Cross-entropy#Cross-entropy_loss_function_and_logistic_regression" title="Cross-entropy">Logarithmic Loss</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Cluster_analysis" title="Cluster analysis">Clustering</a></th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Silhouette_(clustering)" title="Silhouette (clustering)">Silhouette</a></li> <li><a href="/wiki/Calinski-Harabasz_index" class="mw-redirect" title="Calinski-Harabasz index">Calinski-Harabasz index</a></li> <li><a href="/wiki/Davies%E2%80%93Bouldin_index" title="Davies–Bouldin index">Davies-Bouldin</a></li> <li><a href="/wiki/Dunn_index" title="Dunn index">Dunn index</a></li> <li><a href="/wiki/Hopkins_statistic" title="Hopkins statistic">Hopkins statistic</a></li> <li><a href="/wiki/Jaccard_index" title="Jaccard index">Jaccard index</a></li> <li><a href="/wiki/Rand_index" title="Rand index">Rand index</a></li> <li><a href="/wiki/Similarity_measure" title="Similarity measure">Similarity measure</a></li> <li><a href="/wiki/Simple_matching_coefficient" title="Simple matching coefficient">SMC</a></li> <li><a href="/wiki/SimHash" title="SimHash">SimHash</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Ranking_(information_retrieval)" title="Ranking (information retrieval)">Ranking</a></th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Mean_reciprocal_rank" title="Mean reciprocal rank">MRR</a></li> <li><a href="/wiki/NDCG" class="mw-redirect" title="NDCG">NDCG</a></li> <li><a href="/wiki/Average_precision" class="mw-redirect" title="Average precision">AP</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Computer_Vision" class="mw-redirect" title="Computer Vision">Computer Vision</a></th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/PSNR" class="mw-redirect" title="PSNR">PSNR</a></li> <li><a href="/wiki/SSIM" class="mw-redirect" title="SSIM">SSIM</a></li> <li><a href="/wiki/Intersection_over_union" class="mw-redirect" title="Intersection over union">IoU</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Natural_language_processing" title="Natural language processing">NLP</a></th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Perplexity" title="Perplexity">Perplexity</a></li> <li><a href="/wiki/BLEU" title="BLEU">BLEU</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Deep Learning Related Metrics</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Inception_score" title="Inception score">Inception score</a></li> <li><a href="/wiki/Fr%C3%A9chet_inception_distance" title="Fréchet inception distance">FID</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Recommender_system" title="Recommender system">Recommender system</a></th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Coverage_probability" title="Coverage probability">Coverage</a></li> <li><a href="/w/index.php?title=Intra-list_Similarity&action=edit&redlink=1" class="new" title="Intra-list Similarity (page does not exist)">Intra-list Similarity</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Similarity</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cosine_similarity" title="Cosine similarity">Cosine similarity</a></li> <li><a href="/wiki/Euclidean_distance" title="Euclidean distance">Euclidean distance</a></li> <li><a href="/wiki/Pearson_correlation_coefficient" title="Pearson correlation coefficient">Pearson correlation coefficient</a></li></ul> </div></td></tr><tr><td class="navbox-abovebelow hlist" colspan="2"><div> <ul><li><a href="/wiki/Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235"></div><div role="navigation" class="navbox authority-control" aria-label="Navbox" style="padding:3px"><table class="nowraplinks hlist navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Help:Authority_control" title="Help:Authority control">Authority control databases</a>: National <span class="mw-valign-text-top noprint" typeof="mw:File/Frameless"><a href="https://www.wikidata.org/wiki/Q1107106#identifiers" title="Edit this at Wikidata"><img alt="Edit this at Wikidata" src="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/10px-OOjs_UI_icon_edit-ltr-progressive.svg.png" decoding="async" width="10" height="10" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/15px-OOjs_UI_icon_edit-ltr-progressive.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/20px-OOjs_UI_icon_edit-ltr-progressive.svg.png 2x" data-file-width="20" data-file-height="20" /></a></span></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"><ul><li><span class="uid"><a rel="nofollow" class="external text" href="https://d-nb.info/gnd/7522376-4">Germany</a></span></li></ul></div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Cohen%27s_kappa&oldid=1245316837">https://en.wikipedia.org/w/index.php?title=Cohen%27s_kappa&oldid=1245316837</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Categorical_variable_interactions" title="Category:Categorical variable interactions">Categorical variable interactions</a></li><li><a href="/wiki/Category:Nonparametric_statistics" title="Category:Nonparametric statistics">Nonparametric statistics</a></li><li><a href="/wiki/Category:Inter-rater_reliability" title="Category:Inter-rater reliability">Inter-rater reliability</a></li><li><a href="/wiki/Category:Summary_statistics_for_contingency_tables" title="Category:Summary statistics for contingency tables">Summary statistics for contingency tables</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:Wikipedia_articles_needing_page_number_citations_from_April_2012" title="Category:Wikipedia articles needing page number citations from April 2012">Wikipedia articles needing page number citations from April 2012</a></li><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_matches_Wikidata" title="Category:Short description matches Wikidata">Short description matches Wikidata</a></li><li><a href="/wiki/Category:All_articles_with_unsourced_statements" title="Category:All articles with unsourced statements">All articles with unsourced statements</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_April_2012" title="Category:Articles with unsourced statements from April 2012">Articles with unsourced statements from April 2012</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 12 September 2024, at 08:53<span class="anonymous-show"> (UTC)</span>.</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Cohen%27s_kappa&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-h78vk","wgBackendResponseTime":211,"wgPageParseReport":{"limitreport":{"cputime":"0.734","walltime":"1.014","ppvisitednodes":{"value":4747,"limit":1000000},"postexpandincludesize":{"value":237783,"limit":2097152},"templateargumentsize":{"value":2838,"limit":2097152},"expansiondepth":{"value":19,"limit":100},"expensivefunctioncount":{"value":5,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":130872,"limit":5000000},"entityaccesscount":{"value":1,"limit":400},"timingprofile":["100.00% 748.894 1 -total"," 30.51% 228.489 26 Template:Cite_journal"," 25.74% 192.792 1 Template:Reflist"," 24.37% 182.521 1 Template:Statistics"," 23.98% 179.583 1 Template:Navbox_with_collapsible_groups"," 10.15% 75.987 1 Template:Short_description"," 7.29% 54.632 5 Template:Rp"," 6.52% 48.846 2 Template:Pagetype"," 6.42% 48.079 5 Template:R/superscript"," 6.28% 47.030 2 Template:Fix"]},"scribunto":{"limitreport-timeusage":{"value":"0.417","limit":"10.000"},"limitreport-memusage":{"value":6881264,"limit":52428800},"limitreport-logs":"table#1 {\n}\n"},"cachereport":{"origin":"mw-web.eqiad.main-5dc468848-dh57s","timestamp":"20241122141110","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Cohen's kappa","url":"https:\/\/en.wikipedia.org\/wiki\/Cohen%27s_kappa","sameAs":"http:\/\/www.wikidata.org\/entity\/Q1107106","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q1107106","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2005-04-06T19:39:21Z","dateModified":"2024-09-12T08:53:27Z","headline":"Statistic measuring inter-rater agreement for categorical items"}</script> </body> </html>