Classification assessment methods

<!DOCTYPE html> <html lang="en" xmlns:mml="http://www.w3.org/1998/Math/MathML"> <head>   <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="format-detection" content="telephone=no"> <link rel="canonical" href="https://www.emerald.com/insight/content/doi/10.1016/j.aci.2018.08.003/full/html" /> <meta name="citation_xml_url" content="https://www.emerald.com/insight/content/doi/10.1016/j.aci.2018.08.003/full/xml"/>  <meta name="csrf-token" content="RkcahSFU1DO5hR0EtYgmtMNB3mNtDUbkgiL6rQgd"> <title> Classification assessment methods | Emerald Insight</title> <meta name="description" content="1"> <script type="application/ld+json">{ "@type": "ItemPage", "breadcrumb": { "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": 1, "name": "Journals", "item": "https://www.emerald.com/insight/sitemap/publications#journals" }, { "@type": "ListItem", "position": 2, "name": "Applied Computing and Informatics", "item": "https://www.emerald.com/insight/publication/issn/2634-1964" }, { "@type": "ListItem", "position": 3, "name": "Volume 17 Issue 1", "item": "https://www.emerald.com/insight/publication/issn/2634-1964/vol/17/iss/1" }, { "@type": "ListItem", "position": 4, "name": "Classification assessment methods" } ] }, "mainEntity": { "@type": "ScholarlyArticle", "@id": "#article", "url": "https://www.emerald.com/insight/content/doi/10.1016/j.aci.2018.08.003/full/html", "name": "Classification assessment methods", "headline": "Classification assessment methods", "datePublished": "2020-07-30T00:00:00Z", "description": "Classification techniques have been applied to many applications in various fields of sciences. There are several ways of evaluating classification algorithms. The analysis of such metrics and its significance must be interpreted correctly for evaluating different learning algorithms. Most of these measures are scalar metrics and some of them are graphical methods. This paper introduces a detailed overview of the classification assessment measures with the aim of providing the basics of these measures and to show how it works to serve as a comprehensive source for researchers who are interested in this field. This overview starts by highlighting the definition of the confusion matrix in binary and multi-class classification problems. Many classification measures are also explained in details, and the influence of balanced and imbalanced data on each metric is presented. An illustrative example is introduced to show (1) how to calculate these measures in binary and multi-class classification problems, and (2) the robustness of some measures against balanced and imbalanced data. Moreover, some graphical measures such as Receiver operating characteristics (ROC), Precision-Recall, and Detection error trade-off (DET) curves are presented with details. Additionally, in a step-by-step approach, different numerical examples are demonstrated to explain the preprocessing steps of plotting ROC, PR, and DET curves.", "author": [ { "@type": "Person", "name": "Alaa Tharwat" } ], "pageStart": 168, "pageEnd": 192, "isPartOf": "/insight/publication/issn/2210-8327/vol/17/iss/1#issue", "publisher": { "@type": "Organization", "name": "Emerald Publishing Limited" }, "citation": [ { "@type": "Book", "name": "Uci repository of machine learning databases", "url": "http://www.ics.uci.edu/%7Emlearn/MLRepository.html", "author": [ { "@type": "Person", "name": "C. Blake" } ], "datePublished": "1998" }, { "@type": "Book", "name": "Guide to biometrics", "author": [ { "@type": "Person", "name": "R.M. Bolle" }, { "@type": "Person", "name": "J.H. Connell" }, { "@type": "Person", "name": "S. Pankanti" }, { "@type": "Person", "name": "N.K. Ratha" }, { "@type": "Person", "name": "A.W. Senior" } ], "publisher": { "@type": "Organization", "name": "Springer Science & Business Media" }, "datePublished": "2013" }, { "@type": "ScholarlyArticle", "name": "Optimal classifier for imbalanced data using matthews correlation coefficient metric", "headline": "Optimal classifier for imbalanced data using matthews correlation coefficient metric", "isPartOf": { "@type": "PublicationIssue", "issueNumber": 6, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 12, "isPartOf": { "@type": "Periodical", "name": "PLoS One" } } }, "author": [ { "@type": "Person", "name": "S. Boughorbel" }, { "@type": "Person", "name": "F. Jarray" }, { "@type": "Person", "name": "M. El-Anbari" } ], "datePublished": "2017" }, { "@type": "ScholarlyArticle", "name": "The use of the area under the roc curve in the evaluation of machine learning algorithms", "headline": "The use of the area under the roc curve in the evaluation of machine learning algorithms", "pageStart": 1145, "pageEnd": 1159, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 7, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 30, "isPartOf": { "@type": "Periodical", "name": "Pattern Recogn" } } }, "author": [ { "@type": "Person", "name": "A.P. Bradley" } ], "datePublished": "1997" }, { "@type": "Chapter", "name": "The relationship between precision-recall and roc curves", "pageStart": 233, "pageEnd": 240, "isPartOf": { "@type": "Book", "name": "Proceedings of the 23rd International Conference on Machine Learning", "author": [ { "@type": "Person", "name": "J. Davis" }, { "@type": "Person", "name": "M. Goadrich" } ], "publisher": { "@type": "Organization", "name": "ACM" }, "datePublished": "2006" } }, { "@type": "ScholarlyArticle", "name": "Diagnostic tests 4: likelihood ratios", "headline": "Diagnostic tests 4: likelihood ratios", "pageStart": 168, "pageEnd": 169, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 7458, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 329, "isPartOf": { "@type": "Periodical", "name": "Brit. Med. J." } } }, "author": [ { "@type": "Person", "name": "J.J. Deeks" }, { "@type": "Person", "name": "D.G. Altman" } ], "datePublished": "2004" }, { "@type": "Book", "name": "Pattern Classification", "author": [ { "@type": "Person", "name": "R.O. Duda" }, { "@type": "Person", "name": "P.E. Hart" }, { "@type": "Person", "name": "D.G. Stork" } ], "publisher": { "@type": "Organization", "name": "Wiley", "location": "New York" }, "datePublished": "2001" }, { "@type": "ScholarlyArticle", "name": "An introduction to roc analysis", "headline": "An introduction to roc analysis", "pageStart": 861, "pageEnd": 874, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 8, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 27, "isPartOf": { "@type": "Periodical", "name": "Pattern Recogn. Lett." } } }, "author": [ { "@type": "Person", "name": "T. Fawcett" } ], "datePublished": "2006" }, { "@type": "Chapter", "name": "Theoretical analysis of a performance measure for imbalanced data", "pageStart": 617, "pageEnd": 620, "isPartOf": { "@type": "Book", "name": "20th International Conference on Pattern Recognition (ICPR)", "author": [ { "@type": "Person", "name": "V. Garcia" }, { "@type": "Person", "name": "R.A. Mollineda" }, { "@type": "Person", "name": "J.S. Sanchez" } ], "publisher": { "@type": "Organization", "name": "IEEE" }, "datePublished": "2010" } }, { "@type": "ScholarlyArticle", "name": "A simple generalisation of the area under the roc curve for multiple class classification problems", "headline": "A simple generalisation of the area under the roc curve for multiple class classification problems", "pageStart": 171, "pageEnd": 186, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 2, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 45, "isPartOf": { "@type": "Periodical", "name": "Mach. Learn." } } }, "author": [ { "@type": "Person", "name": "D.J. Hand" }, { "@type": "Person", "name": "R.J. Till" } ], "datePublished": "2001" }, { "@type": "ScholarlyArticle", "name": "Learning from imbalanced data", "headline": "Learning from imbalanced data", "pageStart": 1263, "pageEnd": 1284, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 9, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 21, "isPartOf": { "@type": "Periodical", "name": "IEEE Trans. Knowledge Data Eng." } } }, "author": [ { "@type": "Person", "name": "H. He" }, { "@type": "Person", "name": "E.A. Garcia" } ], "datePublished": "2009" }, { "@type": "ScholarlyArticle", "name": "An insight into classification with imbalanced data: empirical results and current trends on using data intrinsic characteristics", "headline": "An insight into classification with imbalanced data: empirical results and current trends on using data intrinsic characteristics", "pageStart": 113, "pageEnd": 141, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 250, "isPartOf": { "@type": "Periodical", "name": "Inf. Sci." } }, "author": [ { "@type": "Person", "name": "V. L\u00f3pez" }, { "@type": "Person", "name": "A. Fern\u00e1ndez" }, { "@type": "Person", "name": "S. Gar\u0107a" }, { "@type": "Person", "name": "V. Palade" }, { "@type": "Person", "name": "F. Herrera" } ], "datePublished": "2013" }, { "@type": "ScholarlyArticle", "name": "Adjusted f-measure and kernel scaling for imbalanced data learning", "headline": "Adjusted f-measure and kernel scaling for imbalanced data learning", "pageStart": 331, "pageEnd": 341, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 257, "isPartOf": { "@type": "Periodical", "name": "Inf. Sci." } }, "author": [ { "@type": "Person", "name": "A. Maratea" }, { "@type": "Person", "name": "A. Petrosino" }, { "@type": "Person", "name": "M. Manzo" } ], "datePublished": "2014" }, { "@type": "ScholarlyArticle", "name": "Comparison of the predicted and observed secondary structure of t4 phage lysozyme", "headline": "Comparison of the predicted and observed secondary structure of t4 phage lysozyme", "pageStart": 442, "pageEnd": 451, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 2, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 405, "isPartOf": { "@type": "Periodical", "name": "Biochim. Biophys. Acta" } } }, "author": [ { "@type": "Person", "name": "B.W. Matthews" } ], "datePublished": "1975" }, { "@type": "Chapter", "name": "Basic principles of roc analysis", "pageStart": 283, "pageEnd": 298, "isPartOf": { "@type": "Book", "name": "Seminars in nuclear medicine", "author": [ { "@type": "Person", "name": "C.E. Metz" } ], "publisher": { "@type": "Organization", "name": "Elsevier" }, "datePublished": "1978" } }, { "@type": "ScholarlyArticle", "name": "Evaluation: from precision, recall and f-measure to roc", "headline": "Evaluation: from precision, recall and f-measure to roc", "pageStart": 37, "pageEnd": 63, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 1, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 2, "isPartOf": { "@type": "Periodical", "name": "informedness, markedness and correlation" } } }, "author": [ { "@type": "Person", "name": "D.M. Powers" } ], "datePublished": "2011" }, { "@type": "ScholarlyArticle", "name": "The precision-recall plot is more informative than the roc plot when evaluating binary classifiers on imbalanced datasets", "headline": "The precision-recall plot is more informative than the roc plot when evaluating binary classifiers on imbalanced datasets", "isPartOf": { "@type": "PublicationIssue", "issueNumber": 3, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 10, "isPartOf": { "@type": "Periodical", "name": "PLoS One" } } }, "author": [ { "@type": "Person", "name": "T. Saito" }, { "@type": "Person", "name": "M. Rehmsmeier" } ], "datePublished": "2015" }, { "@type": "ScholarlyArticle", "name": "Measures derived from a 2 x 2 table for an accuracy of a diagnostic test", "headline": "Measures derived from a 2 x 2 table for an accuracy of a diagnostic test", "pageStart": 1, "pageEnd": 4, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 2, "isPartOf": { "@type": "Periodical", "name": "J. Biometr. Biostat." } }, "author": [ { "@type": "Person", "name": "A. Shaffi" } ], "datePublished": "2011" }, { "@type": "ScholarlyArticle", "name": "Measures derived from a 2 x 2 table for an accuracy of a diagnostic test", "headline": "Measures derived from a 2 x 2 table for an accuracy of a diagnostic test", "pageStart": 128, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 2, "isPartOf": { "@type": "Periodical", "name": "J. Biometr. Biostat." } }, "author": [ { "@type": "Person", "name": "S. Shaikh" } ], "datePublished": "2011" }, { "@type": "Chapter", "name": "Beyond accuracy, f-score and roc: a family of discriminant measures for performance evaluation", "pageStart": 1015, "pageEnd": 1021, "isPartOf": { "@type": "Book", "name": "Australasian Joint Conference on Artificial Intelligence", "author": [ { "@type": "Person", "name": "M. Sokolova" }, { "@type": "Person", "name": "N. Japkowicz" }, { "@type": "Person", "name": "S. Szpakowicz" } ], "publisher": { "@type": "Organization", "name": "Springer" }, "datePublished": "2006" } }, { "@type": "ScholarlyArticle", "name": "A systematic analysis of performance measures for classification tasks", "headline": "A systematic analysis of performance measures for classification tasks", "pageStart": 427, "pageEnd": 437, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 4, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 45, "isPartOf": { "@type": "Periodical", "name": "Inf. Process. Manage." } } }, "author": [ { "@type": "Person", "name": "M. Sokolova" }, { "@type": "Person", "name": "G. Lapalme" } ], "datePublished": "2009" }, { "@type": "Book", "name": "Note on the location of optimal classifiers in n-dimensional roc space", "author": [ { "@type": "Person", "name": "A. Srinivasan" } ], "datePublished": "1999" }, { "@type": "ScholarlyArticle", "name": "Principal component analysis-a tutorial", "headline": "Principal component analysis-a tutorial", "pageStart": 197, "pageEnd": 240, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 3, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 3, "isPartOf": { "@type": "Periodical", "name": "Int. J. Appl. Pattern Recogn." } } }, "author": [ { "@type": "Person", "name": "A. Tharwat" } ], "datePublished": "2016" }, { "@type": "ScholarlyArticle", "name": "Chaotic antlion algorithm for parameter optimization of support vector machine", "headline": "Chaotic antlion algorithm for parameter optimization of support vector machine", "pageStart": 670, "pageEnd": 686, "isPartOf": { "@type": "PublicationIssue", "issueNumber": 3, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 48, "isPartOf": { "@type": "Periodical", "name": "Appl. Intelligence" } } }, "author": [ { "@type": "Person", "name": "A. Tharwat" }, { "@type": "Person", "name": "A.E. Hassanien" } ], "datePublished": "2018" }, { "@type": "ScholarlyArticle", "name": "Classification of toxicity effects of biotransformed hepatic drugs using whale optimized support vector machines", "headline": "Classification of toxicity effects of biotransformed hepatic drugs using whale optimized support vector machines", "pageStart": 132, "pageEnd": 149, "isPartOf": { "@type": "PublicationVolume", "volumeNumber": 68, "isPartOf": { "@type": "Periodical", "name": "J. Biomed. Inf." } }, "author": [ { "@type": "Person", "name": "A. Tharwat" }, { "@type": "Person", "name": "Y.S. Moemen" }, { "@type": "Person", "name": "A.E. Hassanien" } ], "datePublished": "2017" }, { "@type": "WebPage", "name": "Receiver operating characteristic (roc) literature research", "url": "http://splweb.bwh.harvard.edu8000", "author": [ { "@type": "Person", "name": "K.H. Zou" } ], "datePublished": "2002" } ], "mainEntityOfPage": "https://www.emerald.com/insight/content/doi/10.1016/j.aci.2018.08.003/full/html" }, "@context": "https://schema.org/" }</script>  <link rel="schema.DC" href="http://purl.org/DC/elements/1.0/"> <meta name="dc.Title" content="Classification assessment methods"> <meta name="citation_journal_title" content="Applied Computing and Informatics"> <meta name="citation_volume" content="17"> <meta name="citation_issue" content="1"> <meta name="citation_firstpage" content="168"> <meta name="citation_lastpage" content="192"> <meta name="dc.Subject" content="Receiver operating characteristics (ROC),Confusion matrix,Precision-Recall (PR) curve,Classification,Assessment methods"> <meta name="dc.Description" content="Classification techniques have been applied to many applications in various fields of sciences. There are several ways of evaluating classification algorithms. The analysis of such metrics and its significance must be interpreted correctly for evaluating different learning algorithms. Most of these measures are scalar metrics and some of them are graphical methods. This paper introduces a detailed overview of the classification assessment measures with the aim of providing the basics of these measures and to show how it works to serve as a comprehensive source for researchers who are interested in this field. This overview starts by highlighting the definition of the confusion matrix in binary and multi-class classification problems. Many classification measures are also explained in details, and the influence of balanced and imbalanced data on each metric is presented. An illustrative example is introduced to show (1) how to calculate these measures in binary and multi-class classification problems, and (2) the robustness of some measures against balanced and imbalanced data. Moreover, some graphical measures such as Receiver operating characteristics (ROC), Precision-Recall, and Detection error trade-off (DET) curves are presented with details. Additionally, in a step-by-step approach, different numerical examples are demonstrated to explain the preprocessing steps of plotting ROC, PR, and DET curves."> <meta name="dc.Date" scheme="WTN8601" content="2020-07-30T00:00:00Z"> <meta name="dc.Type" content="article"> <meta name="dc.Format" content="text/html"> <meta name="dc.Identifier" scheme="doi" content="10.1016/j.aci.2018.08.003"> <meta name="dc.Identifier" scheme="issn" content="2210-8327"> <meta name="dc.Identifier" scheme="original-pdf" content="j.aci.2018.08.003.pdf"> <meta name="dc.Language" content="en"> <meta name="dc.Coverage" content="world"> <meta name="keywords" content="Receiver operating characteristics (ROC),Confusion matrix,Precision-Recall (PR) curve,Classification,Assessment methods"> <meta name="dc.Publisher" content="Emerald Publishing Limited"> <meta name="DCTERMS.bibliographicCitation" scheme="KEV.ctx" content="&ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.spage=168&rft.epage=192&rft.volume=17&rft.issue=1"> <meta name="dc.Contributor" content="Alaa Tharwat">  <link href="/insight/static/css/app.css?id=d1704eeb3e0eb4acb8985ebfec909296" rel="stylesheet"> <style type="text/css" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> /* hide vue content until vue has initialised */ [v-cloak]{ display: none; } /* reduce CLS on page load */ .search-input-container { height: calc(1.5em + .75rem + 4px); } a.navbar-brand > div.extra-small { height: 1rem; } #breadcrumbs-container { min-height: 2.06667rem; } #search-books-and-journals-container { min-height: calc(1.75rem + 11px); margin-bottom: 1rem; } .header_logo_image { height: calc(14rem / (264/36)); } .toc-alert-button { min-width: calc(13.03472rem + 2.5rem + 14px + 2.405px); min-height: calc(1.75556rem + 2px); } </style> <script type="text/javascript" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> const isReady = (fn) => (document.readyState === "complete" || document.readyState === "interactive") ? setTimeout(fn, 1) : document.addEventListener("DOMContentLoaded", fn) const supportsWebpImages = function () { const elem = document.createElement('canvas'); if (!!(elem.getContext && elem.getContext('2d'))) { // was able or not to get WebP representation return elem.toDataURL('image/webp').indexOf('data:image/webp') === 0; } // very old browser like IE 8, canvas not supported return false; } document.documentElement.classList.add(supportsWebpImages() ? 'webp' : 'no-webp') </script>  <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> window.cookieHelper = { /** * Get the value of a specific cookie * @returns {string | undefined} */ getCookieValue: function (key) { return document.cookie .split(';') .map(function(c) { return c.trim().split('=') }) .filter(function(c) { return c[0] === key }) .map(function(c) { return c[1] })[0] }, cookieExists: function (cookieName) { return document.cookie.split(';').some((item) => item.trim().indexOf(cookieName) == 0) }, setCookie: function (cookieName, value, expires, samesite='lax', path='/') { if(expires == null) { const timeOneDay = 24*60*60*1000 expires = new Date(new Date().getTime() + timeOneDay).toUTCString() } document.cookie = `${cookieName}=${value}; expires=${expires}; samesite=${samesite}; path=${path};` }, deleteCookie: function (cookieName) { document.cookie = `${cookieName}=null; expires=${new Date().getTime() - 24*60*60*1000}; Max-Age=0` }, /** * Get a list of the cookie categories that have been disabled * @returns {string[]} A list of disabled cookie categories */ getDisabledCookies: function () { const disableCookiesValue = this.getCookieValue('disabled_cookies') return disableCookiesValue ? disableCookiesValue .split(',') .map(function(c) { return c.trim() }) : [] }, arePerformanceCookiesEnabled: function () { return this.getDisabledCookies().indexOf('performance') < 0 }, areFunctionalCookiesEnabled: function () { return this.getDisabledCookies().indexOf('functional') < 0 } } </script>   <script type="text/javascript" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ==">(function(c,a){if(!a.__SV){var b=window;try{var d,m,j,k=b.location,f=k.hash;d=function(a,b){return(m=a.match(RegExp(b+"=([^&]*)")))?m[1]:null};f&&d(f,"state")&&(j=JSON.parse(decodeURIComponent(d(f,"state"))),"mpeditor"===j.action&&(b.sessionStorage.setItem("_mpcehash",f),history.replaceState(j.desiredHash||"",c.title,k.pathname+k.search)))}catch(n){}var l,h;window.mixpanel=a;a._i=[];a.init=function(b,d,g){function c(b,i){var a=i.split(".");2==a.length&&(b=b[a[0]],i=a[1]);b[i]=function(){b.push([i].concat(Array.prototype.slice.call(arguments,0)))}}var e=a;"undefined"!==typeof g?e=a[g]=[]:g="mixpanel";e.people=e.people||[];e.toString=function(b){var a="mixpanel";"mixpanel"!==g&&(a+="."+g);b||(a+=" (stub)");return a};e.people.toString=function(){return e.toString(1)+".people (stub)"};l="disable time_event track track_pageview track_links track_forms track_with_groups add_group set_group remove_group register register_once alias unregister identify name_tag set_config reset opt_in_tracking opt_out_tracking has_opted_in_tracking has_opted_out_tracking clear_opt_in_out_tracking people.set people.set_once people.unset people.increment people.append people.union people.track_charge people.clear_charges people.delete_user people.remove".split(" ");for(h=0;h<l.length;h++)c(e,l[h]);var f="set set_once union unset remove delete".split(" ");e.get_group=function(){function a(c){b[c]=function(){call2_args=arguments;call2=[c].concat(Array.prototype.slice.call(call2_args,0));e.push([d,call2])}}for(var b={},d=["get_group"].concat(Array.prototype.slice.call(arguments,0)),c=0;c<f.length;c++)a(f[c]);return b};a._i.push([b,d,g])};a.__SV=1.2;b=c.createElement("script");b.type="text/javascript";b.async=!0;b.crossOrigin='anonymous';b.src="undefined"!==typeof MIXPANEL_CUSTOM_LIB_URL?MIXPANEL_CUSTOM_LIB_URL:"file:"===c.location.protocol&&"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js".match(/^\/\//)?"https://cdn.mxpnl.com/libs/mixpanel-2-latest.min.js":"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js";d=c.getElementsByTagName("script")[0];d.parentNode.insertBefore(b,d)}})(document,window.mixpanel||[]);</script> <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> let isAuthenticatedPerson = false; let mixpanelIdentity = null; const getMixpanelId = () => { const urlParams = new URLSearchParams(window.location.search); const inputId = urlParams.get('distinct_id'); if (inputId) { // Identify with the passed in distinct id mixpanel.register({'distinct_id' : inputId}); // Remove param from url so that if user bookmarks / shared page they would not reuse the id. urlParams.delete('distinct_id'); window.history.pushState('',document.title, window.location.href.replace(window.location.search, '?' + urlParams)) } } const addMixpanelId = () => { // Some ad blockers and privacy extensions block mixpanel being downloaded, so check this function exists before calling it if (typeof(mixpanel.get_distinct_id) === 'function') { const mixpanelId = mixpanel.get_distinct_id(); // Find all mixpanel_link classes and append distinct_id query param to the end [...document.querySelectorAll('.mixpanel_link')] .forEach(node => { const currentUrl = new URL(node.href) currentUrl.searchParams.delete('distinct_id'); currentUrl.searchParams.append('distinct_id', mixpanelId); node.href = currentUrl.href; }) } } // Parse and register last touch UTM tags as super properties. Also add first touch attribution data as a user profile property. function addUtmParams () { const urlParams = new URLSearchParams(window.location.search) const params = {} const firstParams = {} const campaignKeywords = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term'] campaignKeywords.forEach(keyword => { if (!urlParams.has(keyword)) return const kw = urlParams.get(keyword) params[keyword] = kw params[`${keyword} [last touch]`] = kw firstParams[`${keyword} [first touch]`] = kw }) //register last touch UTM mixpanel.people.set(params) mixpanel.register(params) //register first touch UTM mixpanel.people.set_once(firstParams) mixpanel.register_once(firstParams) } const syncMixPanel = () => { getMixpanelId(); addMixpanelId(); addUtmParams(); } mixpanelEvents = { hasReturned: false, stack: [] } function mixpanelUpdateUserDataForEvent(eventName) { if (!isAuthenticatedPerson) { return; } if (['Download', 'Turnaway', 'Saved Search'].includes(eventName)) { mixpanel.people.increment('# of ' + eventName + 's'); } if (eventName === 'Download') { mixpanel.people.set({ 'Last Download Date': new Date() }); mixpanel.identify(mixpanelIdentity); } } function storeOrPushMixpanelEvent(eventType, eventName, data) { if (mixpanelEvents.hasReturned) { mixpanel.push([ eventType, eventName, data ]); } else { mixpanelEvents.stack.push({ eventType, eventName, data }); } } function pushStoredMixpanelEvents () { var sizeOfTracker = mixpanelEvents.stack.length; mixpanelEvents.hasReturned = true; for (var i = 0; i < sizeOfTracker; i++) { var ev = mixpanelEvents.stack[i]; if (typeof ev.eventName === 'undefined') continue; mixpanel.push([ev.eventType, ev.eventName, ev.data]); // Update the mixpanel data now that the user is authenticated. mixpanelUpdateUserDataForEvent(ev.eventName); } } /** * Set up mixpanel to track a link click. This allows mixpanel to make sure the click event is tracked before * a new page is navigated to * Note: This doesn't handle right click -> open in new tab/window. There appears to be no straightforward way to handle * that short of rewriting the links to point to an internal redirect endpoint and handling the mixpanel event there * @param element {HTMLElement} The link element * @param eventName {string} The name of the mixpanel event * @param makeData {function} A function that takes the element and returns the mixpanel properties for the event */ function mixpanelTrackLinkClick(element, eventName, makeData) { // use built-in mixpanel link tracking - this tracks left clicks and enter key mixpanel.track_links(element, eventName, makeData) // handle middle clicks - shortcut for open in new tab element.addEventListener('mousedown', event => { if (event.button === 1) { storeOrPushMixpanelEvent('track', eventName, makeData(event.currentTarget)) } }) } const mixpanelOptions = { api_host: "https://api.mixpanel.com", cookie_expiration: 30, loaded: syncMixPanel, debug: false} mixpanel.init('8e8ac8d2dbd2378f29bd1dd9116a0c9a', mixpanelOptions); </script>    <style nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ==">.async-hide { opacity: 0 !important} </style> <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> (function(a,s,y,n,c,h,i,d,e) { s.className+=' '+y; h.start=1*new Date; h.end=i=function(){s.className=s.className.replace(RegExp(' ?'+y),'')}; (a[n]=a[n]||[]).hide=h; setTimeout(function(){i();h.end=null},c); h.timeout=c; }) (window,document.documentElement,'async-hide','dataLayer',4000,{'GTM-KGFMJXR':true}); </script>  <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ==" data-nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ==" id="GTM-container">(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;var n=d.querySelector('[nonce]');n&&j.setAttribute('nonce',n.nonce||n.getAttribute('nonce'));f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KGFMJXR');</script>  <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> ((disableGaKey) => { window[disableGaKey] = !window.cookieHelper.arePerformanceCookiesEnabled() isReady(() => { window.events.$on('cookiesUpdated', () => { window[disableGaKey] = !window.cookieHelper.arePerformanceCookiesEnabled() }) }) })('ga-disable-UA-127317715-1') </script>  <script type="text/javascript" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.4/MathJax.js?config=TeX-MML-AM_SVG" integrity="sha512-xAWI9i8WMRLdgksuhaMCYMTw9D+MEc2cYVBApWwGRJ0cdcywTjMovOfJnlGt9LlEQj6QzyMzpIZLMYujetPcQg==" crossorigin="anonymous"></script> <script src="https://kit.fontawesome.com/9d150393c6.js" crossorigin="anonymous" defer></script> <script src="https://scholar.google.com/scholar_js/casa.js" async></script> </head> <body data-spy="scroll" data-target=".table-of-contents" data-offset="220" class="">  <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KGFMJXR" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>  <div id="cookies-consent"> <cookies-manager banner-class-name="" cookie-policy-url="/insight/site-policies/cookie-policy" root="/insight/" ></cookies-manager> </div> <div id="app" class=" "> <header class="mb-0 mb-lg-0 page-header "> <div id="support-banner-container" style="display:inline"> <support-banner root="/insight/"></support-banner> </div> <div class="d-none"> <div id="browse-content-template"> <nav class="intent_browse_content w-100 medium bg-aquamarine-menu"> <div class="justify-content-around d-flex py-2"> <a href="/insight/browse/publications">Books and journals</a> <a href="/insight/content/case-studies">Case studies</a> <a href="/insight/content/briefings">Expert Briefings</a> <a href="/insight/content/open-access">Open Access</a> </div> </nav> </div> </div> <div class="d-none"> <div id="publish-with-us-template"> <a href="https://www.emeraldgrouppublishing.com/publish-with-us" aria-label="This link opens a new tab where you can access information on how to publish with Emerald." title="This link opens a new tab where you can access information on how to publish with Emerald." target="_blank"> Publish with us <span role="img" aria-hidden="true" class="far fa-external-link ml-1 text-primary"></span> </a> </div> </div> <div id="login-header-container"> <page-header id="login-header" theme="light" root="/insight/" defaultusername="Guest user" reloadpageonlogin="false" routes="{"browseBooksAndJournals":"\/insight\/browse\/publications","caseStudies":"\/insight\/content\/case-studies","open-access":"\/insight\/content\/open-access","briefings":"\/insight\/content\/briefings","logout":"\/insight\/logout","login":"\/insight\/login","register":"\/insight\/register","savedSearches":"\/insight\/savedSearches","myProducts":"\/insight\/products","myProfile":"\/insight\/profile","publishWithUs":"https:\/\/www.emeraldgrouppublishing.com\/publish-with-us","home":"\/insight"}" opencartbaseurl="https://shop.emerald.com/" show_cart_menu="1" account="" additionalaccount = "" country_code="SG"> <template v-slot:default="props"> <meta name="EZproxyRewriting" content="disable"> <welcome-message class="bg-pale-blue text-dark topHeader" message="Welcome " :banner-message-array="props.welcomeTexts" :is-administrator="props.account.isAdministrator" amp-url="https://manage.emerald.com/"> </welcome-message> <meta name="EZproxyRewriting" content="enable"> </template> </page-header> </div> <div class="nav-div nav-div-search bg-primary py-1 py-md-2 text-dark"> <div class="container d-sm-flex d-flex-row align-items-center"> <div class="flex-grow-1 search-input-container"> <search-content action="/insight/search" advancedsearch="/insight/advanced-search" publicationurl="/insight/publication" value="" placeholder="Enter your search terms here" autocomplete="/insight/search/title" v-bind:errors="[]" v-bind:showall="true" v-bind:openaccess="false" v-bind:home="false"> </search-content> </div> <div class="d-flex d-flex-row align-items-center justify-content-end"> <div class="pr-0 small mr-0 my-0 text-center text-right text-md-left advanced-search-text"> <a class="intent_AdvancedSearch font-size-small btn btn-link rounded-0 px-2 py-0 py-md-2 text-white" href="/insight/advanced-search">Advanced <br class="d-none d-sm-inline d-md-none" />search</a> </div> </div> </div> </div> <div id="alert-notification-container"> <alert-notification></alert-notification> </div> </header> <main role="main" class="main-content" id="mainContent" tabindex="0"> <div class="ml-0 w-100 bg-white"> <div id="breadcrumbs" class="container"> <div class="row ml-0" id="breadcrumbs-container"> <breadcrumbs root="/insight/" crumbs="[["Journals","\/sitemap\/publications#journals"],["Applied Computing and Informatics","\/publication\/issn\/2634-1964"],["Volume 17 Issue 1","\/publication\/issn\/2634-1964\/vol\/17\/iss\/1"],["Classification assessment methods",""]]"> </breadcrumbs> </div> </div> </div> <div class="container content_block"> <div class="row"> <div class="col-12 col-md-2"></div> <div class="col-12 col-md-10"> <header class="py-3"> <h1 class="content-title intent_article_title mt-0 mb-3">Classification assessment methods</h1> <section id="intent_contributors" class="mt-4 intent_contributors"> <contributor-block block="<span class="m:contributor-display"><div class="contrib_block__contrib intent_contributor" contrib-type="author" corresp="yes"><a rel="nofollow" class="contrib-search" href="/insight/search?q=Alaa Tharwat" aria-label="Search for more content by contributor Alaa Tharwat" name-style="western"> <span class="given-names">Alaa</span> <span class="surname">Tharwat</span> </a><span class="contrib_block__aff--italic intent_contributor_affiliate"> (Faculty of Computer Science and Engineering, <span class="institution-wrap"><span class="institution">Frankfurt University of Applied Sciences</span></span>, Frankfurt, <span class="country" country="DE">Germany</span>) </span></div></span>" contrib-count="1" affil-count="1"> <span class="m:contributor-display"><div class="contrib_block__contrib intent_contributor" contrib-type="author" corresp="yes"><a rel="nofollow" class="contrib-search" href="/insight/search?q=Alaa Tharwat" aria-label="Search for more content by contributor Alaa Tharwat" name-style="western"> <span class="given-names">Alaa</span> <span class="surname">Tharwat</span> </a><span class="contrib_block__aff--italic intent_contributor_affiliate"> (Faculty of Computer Science and Engineering, <span class="institution-wrap"><span class="institution">Frankfurt University of Applied Sciences</span></span>, Frankfurt, <span class="country" country="DE">Germany</span>) </span></div></span> </contributor-block> </section> <div class="mb-2 row"> <div class="col-12 col-md-6"> <p> <span class="intent_journal_title"><a href="/insight/publication/issn/2634-1964">Applied Computing and Informatics</a></span> </p> <p class="mt-0"> <span> <abbr title="International Standard Serial Number." class="font-weight-normal">ISSN</abbr>: <span class="intent_journal_issn">2634-1964</span> </span> </p> <p class="mt-0"> <span role="img" aria-hidden="true" class="fas open-access-icon mr-2" title="Open Access."></span> <span class="intent_journal_publication_date">Article publication date: 30 July 2020</span> </p> <p class="mt-0"> <span class="intent_journal_print_date">Issue publication date: 4 January 2021</span> </p> </div> <div class="col-12 col-md-6 d-flex mt-3 mt-md-0"> <div class="d-flex flex-column align-items-center pr-3"> <label class="mini-caps" id="downloadCountLabel-1">Downloads</label> <div class="text-primary pt-2 d-flex align-items-center intent_download_counts_badge" aria-labelledby="downloadCountLabel-1" title="This item has been downloaded 40647 times since 2020." data-toggle="tooltip" data-placement="right" tabindex="0"> <span class="fal fa-file-download fa-2x px-1"></span> <span class="intent_download_count small flex-grow-1 text-center"> 40647 </span> </div> </div> <div class="altmetric-mount"> <altmetric-badge root="/insight/" content-doi="10.1016/j.aci.2018.08.003" publication-doi="10.1108/ACI"> </altmetric-badge> </div> </div> </div> </header> </div> </div> </div> <div class="bg-light border-top border-bottom py-3 mb-3 content_block "> <div class="container "> <div class="row"> <div class="col-12 col-md-8 offset-md-2"> <div class="text-center text-sm-left"> <a href="/insight/content/doi/10.1016/j.aci.2018.08.003/full/pdf?title=classification-assessment-methods" class="intent_pdf_link text-uppercase d-inline-block" target="_blank" aria-label="Download PDF (2.2 MB)."> <span role="img" aria-hidden="true" class="fas fa-file-pdf icon-small"></span> pdf (2.2 MB) </a> <a href="#article" class="intent_article_view_link ml-3 d-none"> <span role="img" aria-hidden="true" class="fas fa-file-alt icon-small"></span> Article view </a> <a href="#figure-view" class="intent_figure_view_link ml-3 d-none"> <span role="img" aria-hidden="true" class="fas fa-images icon-small"></span> Figure view </a> <a href="#citation" class="intent_cited ml-3 d-none"> <span role="img" aria-hidden="true" class="fas fa-comments icon-small"></span> Cited (<span class="intent_cited_count">13</span>) </a> <a href="#cite" class="intent_cite_link btn btn-outline-primary ml-3 d-none text-uppercase">cite article </a> </div> </div> </div> </div> </div> <div class="container content_block"> <div class="row"> <div class="col-12 col-md-2 col-sm-12 p-0">  <div id="tocscroll" > <toc-scroll :headers="{"sec001":"1. Introduction","sec002":"2. Classification performance","sec003":"3. Receiver operating characteristics (ROC)","sec004":"4. Area under the ROC curve (AUC)","sec005":"5. Precision-Recall (PR) curve","sec006":"6. Biometrics measures","sec007":"7. Experimental results","sec008":"8. Conclusions"}" :abstractcontent="true"></toc-scroll> </div>  </div> <div class="col-12 col-md-7"> <div id="articleContent" class="tab-content" aria-live="polite"> <div class="tab-pane active my-2" role="tabpanel" aria-hidden="false" aria-labelledby="articles" tabindex="0" id="article-tab" > <section id="abstract" class="intent_abstract mb-5 Abstract" tabindex="0"> <h2 class="intent_title Abstract__title mb-1">Abstract</h2> <div class="intent_sub_item Abstract__block"> <h3 class="intent_sub_title Abstract__block__title mb-1 mt-3"></h3> <section class="intent_sub_content Abstract__block__text"> <p>Classification techniques have been applied to many applications in various fields of sciences. There are several ways of evaluating classification algorithms. The analysis of such metrics and its significance must be interpreted correctly for evaluating different learning algorithms. Most of these measures are scalar metrics and some of them are graphical methods. This paper introduces a detailed overview of the classification assessment measures with the aim of providing the basics of these measures and to show how it works to serve as a comprehensive source for researchers who are interested in this field. This overview starts by highlighting the definition of the confusion matrix in binary and multi-class classification problems. Many classification measures are also explained in details, and the influence of balanced and imbalanced data on each metric is presented. An illustrative example is introduced to show (1) how to calculate these measures in binary and multi-class classification problems, and (2) the robustness of some measures against balanced and imbalanced data. Moreover, some graphical measures such as Receiver operating characteristics (ROC), Precision-Recall, and Detection error trade-off (DET) curves are presented with details. Additionally, in a step-by-step approach, different numerical examples are demonstrated to explain the preprocessing steps of plotting ROC, PR, and DET curves.</p> </section> </div> </section> <section id="keywords_list" class="intent_keywords"> <h2 class="intent_title" id="page__keywords-label">Keywords</h2> <ul class="intent_paragraph list-inline" aria-labelledby="page__keywords-label"> <li class="list-inline-item mr-0"> <a rel="nofollow" href="/insight/search?q=Receiver+operating+characteristics+%28ROC%29"class="intent_link btn btn-outline-secondary btn-sm my-1"title="Search for keyword Receiver operating characteristics (ROC)"><span class="intent_text">Receiver operating characteristics (ROC)</span></a></li> <li class="list-inline-item mr-0"> <a rel="nofollow" href="/insight/search?q=Confusion+matrix"class="intent_link btn btn-outline-secondary btn-sm my-1"title="Search for keyword Confusion matrix"><span class="intent_text">Confusion matrix</span></a></li> <li class="list-inline-item mr-0"> <a rel="nofollow" href="/insight/search?q=Precision-Recall+%28PR%29+curve"class="intent_link btn btn-outline-secondary btn-sm my-1"title="Search for keyword Precision-Recall (PR) curve"><span class="intent_text">Precision-Recall (PR) curve</span></a></li> <li class="list-inline-item mr-0"> <a rel="nofollow" href="/insight/search?q=Classification"class="intent_link btn btn-outline-secondary btn-sm my-1"title="Search for keyword Classification"><span class="intent_text">Classification</span></a></li> <li class="list-inline-item mr-0"> <a rel="nofollow" href="/insight/search?q=Assessment+methods"class="intent_link btn btn-outline-secondary btn-sm my-1"title="Search for keyword Assessment methods"><span class="intent_text">Assessment methods</span></a></li> </ul> </section> <section class="Citation mb-2"> <h2 class="h4 intent_citation Citation__title">Citation</h2> <p> <a rel="nofollow" href="/insight/search?q=Alaa Tharwat" title="Alaa Tharwat" class="font-weight-light link-dark">Tharwat, A.</a> (2021), "Classification assessment methods", <i><a href="/insight/publication/issn/2634-1964">Applied Computing and Informatics</a></i>, Vol. 17 No. 1, pp. 168-192. <a class='intent_doi_link Citation__identifier__link' href='https://doi.org/10.1016/j.aci.2018.08.003' title='DOI: https://doi.org/10.1016/j.aci.2018.08.003'>https://doi.org/10.1016/j.aci.2018.08.003</a> </p> <div id="citation-download"> <citation-download doi="10.1016/j.aci.2018.08.003" root="/insight/"/> </div> </section> <section class="mt-1 Body" v-pre> <h2 class="d-inline h4" id="page__publisher-label">Publisher</h2>: <p class="publisher d-inline" aria-labelledby="page__publisher-label"> Emerald Publishing Limited </p> </section> <p class="Citation__identifier"> Copyright <span class="intent_copyright_text">© 2018, Alaa Tharwat</span> </p> <section class="License mt-1 Body "> <h2 class="h4 intent_license">License</h2> <p class="d-inline">Published in <em>Applied Computing and Informatics</em>. Published by Emerald Publishing Limited. This article is published under the Creative Commons Attribution (CC BY 4.0) license. Anyone may reproduce, distribute, translate and create derivative works of this article (for both commercial and non-commercial purposes), subject to full attribution to the original publication and authors. The full terms of this license may be seen at <a href="http://creativecommons.org/licences/by/4.0/legalcode" class="intent_external_link text-link ExtLink" rel="noopener noreferrer nofollow" target="_blank">http://creativecommons.org/licences/by/4.0/legalcode</a></p> </section>  <hr> <section class="mb-5 Body " v-pre> <section tabindex="0" id="sec001"> <h2 class="intent_subheading">1. Introduction</h2><p>Classification techniques have been applied to many applications in various fields of sciences. In classification models, the training data are used for building a classification model to predict the class label for a new sample. The outputs of classification models can be discrete as in the decision tree classifier or continuous as the Naive Bayes classifier [<a href="#ref007" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref007" alt="" title="">7</a>]. However, the outputs of learning algorithms need to be assessed and analyzed carefully and this analysis must be interpreted correctly, so as to evaluate different learning algorithms.</p><p>The classification performance is represented by scalar values as in different metrics such as accuracy, sensitivity, and specificity. Comparing different classifiers using these measures is easy, but it has many problems such as the sensitivity to imbalanced data and ignoring the performance of some classes. Graphical assessment methods such as Receiver operating characteristics (ROC) and Precision-Recall curves give different interpretations of the classification performance.</p><p>Some of the measures which are derived from the confusion matrix for evaluating a diagnostic test are reported in [<a href="#ref019" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref019" alt="" title="">19</a>]. In that paper, only eight measures were introduced. Powers introduced an excellent discussion of the precision, Recall, F-score, ROC, Informedness, Markedness and Correlation assessment methods with details explanations [<a href="#ref016" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016" alt="" title="">16</a>]. Sokolova et al. reported some metrics which are used in medical diagnosis [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. Moreover, a good investigation of some measures and the robustness of these measures against different changes in the confusion matrix are introduced in [<a href="#ref021" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref021" alt="" title="">21</a>]. Tom Fawcett presented a detailed introduction to the ROC curve including (1) good explanations of the basics of the ROC curve, (2) clear example for generating the ROC curve, (3) comprehensive discussions, and (4) good explanations of the Area under curve (AUC) metric [<a href="#ref008" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref008" alt="" title="">8</a>]. Jesse Davis and Mark Goadrich reported the relationship between the ROC and Precision-Recall curves [<a href="#ref005" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref005" alt="" title="">5</a>]. Our paper introduces a detailed overview of the classification assessment methods with the goal of providing the basic principles of these measures and to show how it works to serve as a comprehensive source for researchers who are interested in this field. This paper has details of most of the well-known classification assessment methods. Moreover, this paper introduces (1) the relations between different assessment methods, (2) numerical examples to show how to calculate these assessment methods, (3) the robustness of each method against imbalanced data which is one of the most important problems in real-time applications, and (4) explanations of different curves in a step-by-step approach.</p><p>This paper is divided into eight sections. <a href="#sec002" class="text-link scroll_to Link intent_link" data-target="sec002" alt="" title="">Section 2</a> gives an overview of the classification assessment methods. This section begins by explaining the confusion matrix for binary and multi-class classification problems. Based on the data that can be extracted from the confusion matrix, many classification metrics can be calculated. Moreover, the influence of balanced and imbalanced data on each assessment method is introduced. Additionally, an illustrative numerical example is presented to show (1) how to calculate these measures in both binary and multi-class classification problems, and (2) the robustness of some measures against balanced and imbalanced data. <a href="#sec003" class="text-link scroll_to Link intent_link" data-target="sec003" alt="" title="">Section 3</a> introduces the basics of the ROC curve, which are required for understanding how to plot and interpret it. This section also presents visualized steps with an illustrative example for plotting the ROC curve. The AUC measure is presented in <a href="#sec004" class="text-link scroll_to Link intent_link" data-target="sec004" alt="" title="">Section 4</a>. In this section, the AUC algorithm with detailed steps is explained. <a href="#sec005" class="text-link scroll_to Link intent_link" data-target="sec005" alt="" title="">Section 5</a> presents the basics of the Precision-Recall curve and how to interpret it. Further, in a step-by-step approach, different numerical examples are demonstrated to explain the preprocessing steps of plotting ROC and PR curves in <a href="#sec003 sec005" class="text-link scroll_to Link intent_link" data-target="sec003 sec005" alt="" title="">Sections 3 and 5</a>. Classification assessment methods for biometric models including steps of plotting the DET curve are presented in <a href="#sec006" class="text-link scroll_to Link intent_link" data-target="sec006" alt="" title="">Section 6</a>. In <a href="#sec007" class="text-link scroll_to Link intent_link" data-target="sec007" alt="" title="">Section 7</a>, results in terms of different assessment methods of a simple experiment are presented. Finally, concluding remarks will be given in <a href="#sec008" class="text-link scroll_to Link intent_link" data-target="sec008" alt="" title="">Section 8</a>.</p></section> <section tabindex="0" id="sec002"> <h2 class="intent_subheading">2. Classification performance</h2><p>The assessment method is a key factor in evaluating the classification performance and guiding the classifier modeling. There are three main phases of the classification process, namely, <em>training</em> phase, <em>validation</em> phase, and <em>testing</em> phase. The model is trained using input patterns and this phase is called the training phase. These input patterns are called training data which are used for training the model. During this phase, the parameters of a classification model are adjusted. The training error measures how well the trained model fits the training data. However, the training error always smaller than the testing error and the validation error because the trained model fits the same data which are used in the training phase. The goal of a learning algorithm is to learn from the training data to predict class labels for unseen data; this is in the testing phase. However, the testing error or out-of-sample error cannot be estimated because the class labels or outputs of testing samples are unknown. This is the reason why the validation phase is used for evaluating the performance of the trained model. In the validation phase, the validation data provide an unbiased evaluation of the trained model while tuning the model’s hyperparameters.</p><p>According to the number of classes, there are two types of classification problems, namely, binary classification where there are only two classes, and multi-class classification where the number of classes is higher than two. Assume we have two classes, i.e., binary classification, <em>P</em> for <em>positive</em> class and <em>N</em> for <em>negative</em> class. An unknown sample is classified to <em>P</em> or <em>N</em>. The classification model that was trained in the training phase is used to predict the true classes of unknown samples. This classification model produces continuous or discrete outputs. The discrete output that is generated from a classification model represents the predicted discrete class label of the unknown/test sample, while continuous output represents the estimation of the sample’s class membership probability.</p><p><a href="#F_j.aci.2018.08.003001" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003001" alt="An illustrative example of the 2×2 confusion matrix. There are two true classes P and N. The output of the predicted class is true or false." title="An illustrative example of the 2×2 confusion matrix. There are two true classes P and N. The output of the predicted class is true or false.">Figure 1</a> shows that there are four possible outputs which represent the elements of a <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span> <em>confusion matrix</em> or a <em>contingency table</em>. The green diagonal represents correct predictions and the pink diagonal indicates the incorrect predictions. If the sample is positive and it is classified as positive, i.e., correctly classified positive sample, it is counted as a <em>true positive</em> (<em>TP</em>); if it is classified as negative, it is considered as a <em>false negative</em> (<em>FN</em>) or <em>Type II error</em>. If the sample is negative and it is classified as negative it is considered as <em>true negative</em> (<em>TN</em>); if it is classified as positive, it is counted as <em>false positive</em> (<em>FP</em>), <em>false alarm</em> or <em>Type I error</em>. As we will present in the next sections, the confusion matrix is used to calculate many common classification metrics.</p><p><a href="#F_j.aci.2018.08.003002" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003002" alt="An illustrative example of the confusion matrix for a multi-class classification test." title="An illustrative example of the confusion matrix for a multi-class classification test.">Figure 2</a> shows the confusion matrix for a multi-class classification problem with three classes (A, B, and C). As shown, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>TP</mml:mi></mml:mrow><mml:mi>A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> is the number of true positive samples in class A, i.e., the number of samples that are correctly classified from class A, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AB</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span> is the samples from class A that were incorrectly classified as class B, i.e., misclassified samples. Thus, the false negative in the A class (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi>A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span>) is the sum of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AB</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AC</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span> (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi>A</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AB</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AC</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span>) which indicates the sum of all class A samples that were incorrectly classified as class B or C. Simply, <em>FN</em> of any class which is located in a column can be calculated by adding the errors in that class/column. Whereas the false positive for any predicted class which is located in a row represents the sum of all errors in that row. For example, the false positive in class A (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi>A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span>) is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi>A</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">BA</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi>E</mml:mi><mml:mrow><mml:mi mathvariant="italic">CA</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span>. With <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>×</mml:mo><mml:mi>m</mml:mi></mml:mrow></mml:mrow></mml:math></span> confusion matrix there are <em>m</em> correct classifications and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msup><mml:mi>m</mml:mi><mml:mn>2</mml:mn></mml:msup><mml:mo>−</mml:mo><mml:mi>m</mml:mi></mml:mrow></mml:mrow></mml:math></span> possible errors [<a href="#ref022" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref022" alt="" title="">22</a>].</p><section tabindex="0" id="sec002.1"> <h3 class="intent_subheading">2.1 Classification metrics with imbalanced data</h3><p>Different assessment methods are sensitive to the imbalanced data when the samples of one class in a dataset outnumber the samples of the other class(es) [<a href="#ref025" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref025" alt="" title="">25</a>]. To explain this is so, consider the confusion matrix in <a href="#F_j.aci.2018.08.003001" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003001" alt="An illustrative example of the 2×2 confusion matrix. There are two true classes P and N. The output of the predicted class is true or false." title="An illustrative example of the 2×2 confusion matrix. There are two true classes P and N. The output of the predicted class is true or false.">Figure 1</a>. The class distribution is the ratio between the positive and negative samples (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mi>P</mml:mi><mml:mi>N</mml:mi></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>) represents the relationship between the left column to the right column. Any assessment metric that uses values from both columns will be sensitive to the imbalanced data as reported in [<a href="#ref008" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref008" alt="" title="">8</a>]. For example, some metrics such as accuracy and precision<a href="#fn001" class="text-link scroll_to Link intent_footnote_link Link__footnote" data-target="fn001" alt="" title=""><sup>1</sup></a> use values from both columns of the confusion matrix; thus, as data distributions change, these metrics will change as well, even if the classifier performance does not. Therefore, such these metrics cannot distinguish between the numbers of corrected labels from different classes [<a href="#ref011" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref011" alt="" title="">11</a>]. This fact is partially true because there are some metrics such as Geometric Mean (<em>GM</em>) and Youden’s index (<em>YI</em>)<a href="#fn002" class="text-link scroll_to Link intent_footnote_link Link__footnote" data-target="fn002" alt="" title=""><sup>2</sup></a> use values from both columns and these metrics can be used with balanced and imbalanced data. This can be interpreted as that the metrics which use values from one column cancel the changes in the class distribution. However, some metrics which use values from both columns are not sensitive to the imbalanced data because the changes in the class distribution cancel each other. For example, the accuracy is defined as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span> and the <em>GM</em> is defined as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">GM</mml:mi><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:msqrt><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:msqrt></mml:mrow></mml:mrow></mml:math></span>; thus, both metrics use values from both columns of the confusion matrix. Changing the class distribution can be obtained by increasing/decreasing the number of samples of negative/positive class. With the same classification performance, assume that the negative class samples are increased by <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mi>α</mml:mi></mml:mrow></mml:math></span> times; thus, the <em>TN</em> and <em>FP</em> values will be <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>α</mml:mi><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mrow></mml:math></span>, respectively; thus, the accuracy will be, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>≠</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>. This means that the accuracy is affected by the changes in the class distribution. On the other hand, the <em>GM</em> metric will be, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">GM</mml:mi><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:msqrt><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:msqrt></mml:mrow></mml:mrow></mml:math></span> and hence the changes in the negative class cancel each other. This is the reason why the <em>GM</em> metric is suitable for the imbalanced data. Similarly, any metric can be checked to know if it is sensitive to the imbalanced data or not.</p></section><section tabindex="0" id="sec002.2"> <h3 class="intent_subheading">2.2 Accuracy and error rate</h3><p><em>Accuracy</em> (<em>Acc</em>) is one of the most commonly used measures for the classification performance, and it is defined as a ratio between the correctly classified samples to the total number of samples as follows [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]:<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn1"><span class="label">(1)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m1" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div>where <em>P</em> and <em>N</em> indicate the number of positive and negative samples, respectively.</p><p>The complement of the accuracy metric is the <em>Error rate</em> (<em>ERR</em>) or <em>misclassification rate</em>. This metric represents the number of misclassified samples from both positive and negative classes, and it is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">EER</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>/</mml:mo><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> [<a href="#ref004" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref004" alt="" title="">4</a>]. Both accuracy and error rate metrics are sensitive to the imbalanced data. Another problem with the accuracy is that two classifiers can yield the same accuracy but perform differently with respect to the types of correct and incorrect decisions they provide [<a href="#ref009" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref009" alt="" title="">9</a>]. However, Takaya Saito and Marc Rehmsmeier reported that the accuracy is suitable with imbalanced data because they found that the accuracy values of the balanced and imbalanced data in their example were identical [<a href="#ref017" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref017" alt="" title="">17</a>]. The reason why the accuracy values were identical in their example is that the sum of <em>TP</em> and <em>TN</em> in the balanced and imbalanced data was the same.</p></section><section tabindex="0" id="sec002.3"> <h3 class="intent_subheading">2.3 Sensitivity and specificity</h3><p><em>Sensitivity</em>, <em>True positive rate</em> (<em>TPR</em>), <em>hit rate</em>, or <em>recall</em>, of a classifier represents the positive correctly classified samples to the total number of positive samples, and it is estimated according to <a href="#eqn2" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn2" alt="" title="">Eq. (2)</a> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. Whereas <em>specificity</em>, <em>True negative rate</em> (<em>TNR</em>), or inverse recall is expressed as the ratio of the correctly classified negative samples to the total number of negative samples as in <a href="#eqn2" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn2" alt="" title="">Eq. (2)</a> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. Thus, the specificity represents the proportion of the negative samples that were correctly classified, and the sensitivity is the proportion of the positive samples that were correctly classified. Generally, we can consider sensitivity and specificity as two kinds of accuracy, where the first for actual positive samples and the second for actual negative samples. Sensitivity depends on <em>TP</em> and <em>FN</em> which are in the same column of the confusion matrix, and similarly, the specificity metric depends on <em>TN</em> and <em>FP</em> which are in the same column; hence, both sensitivity and specificity can be used for evaluating the classification performance with imbalanced data [<a href="#ref009" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref009" alt="" title="">9</a>].<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn2"><span class="label">(2)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m2" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">P</mml:mi></mml:mfrac><mml:mo>,</mml:mo><mml:mtext> </mml:mtext><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mi mathvariant="italic">N</mml:mi></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div></p><p>The accuracy can also be defined in terms of sensitivity and specificity as follows [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]:<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn3"><span class="label">(3)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m3" display="block"><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow><mml:mo>=</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">P</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">N</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mfrac><mml:mi mathvariant="italic">P</mml:mi><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac><mml:mfrac><mml:mi mathvariant="italic">N</mml:mi><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:math></span></div></div></p></section><section tabindex="0" id="sec002.4"> <h3 class="intent_subheading">2.4 False positive and false negative rates</h3><p><em>False positive rate</em> (<em>FPR</em>) is also called <em>false alarm rate</em> (<em>FAR</em>), or Fallout, and it represents the ratio between the incorrectly classified negative samples to the total number of negative samples [<a href="#ref016" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016" alt="" title="">16</a>]. In other words, it is the proportion of the negative samples that were incorrectly classified. Hence, it complements the specificity as in <a href="#eqn4" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn4" alt="" title="">Eq. (4)</a> [<a href="#ref021" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref021" alt="" title="">21</a>]. The <em>False negative rate</em> (<em>FNR</em>) or <em>miss rate</em> is the proportion of positive samples that were incorrectly classified. Thus, it complements the sensitivity measure and it is defined in <a href="#eqn5" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn5" alt="" title="">Eq. (5)</a>. Both FPR and FNR are not sensitive to changes in data distributions and hence both metrics can be used with imbalanced data [<a href="#ref009" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref009" alt="" title="">9</a>].<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn4"><span class="label">(4)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m4" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">N</mml:mi></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn5"><span class="label">(5)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m5" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FNR</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi mathvariant="italic">P</mml:mi></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div></p></section><section tabindex="0" id="sec002.5"> <h3 class="intent_subheading">2.5 Predictive values</h3><p>Predictive values (positive and negative) reflect the performance of the prediction. <em>Positive prediction value</em> (<em>PPV</em>) or <em>precision</em> represents the proportion of positive samples that were correctly classified to the total number of positive predicted samples as indicated in <a href="#eqn6" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn6" alt="" title="">Eq. (6)</a> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. On the contrary, <em>Negative predictive value</em> (<em>NPV</em>), inverse precision, or true negative accuracy (<em>TNA</em>) measures the proportion of negative samples that were correctly classified to the total number of negative predicted samples as indicated in <a href="#eqn7" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn7" alt="" title="">Eq. (7)</a> [<a href="#ref016" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016" alt="" title="">16</a>]. These two measures are sensitive to the imbalanced data [<a href="#ref021 ref009" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref021 ref009" alt="" title="">21,9</a>]. <em>False discovery rate</em> (FDR) and <em>False omission rate</em> (FOR) measures complements the PPV and NPV, respectively (see <a href="#eqn6 eqn7" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn6 eqn7" alt="" title="">Eq. (6) and (7)</a>).<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn6"><span class="label">(6)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m6" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>=</mml:mo><mml:mtext>Precision</mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">FDR</mml:mi></mml:mrow></mml:mrow></mml:math></span></div></div><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn7"><span class="label">(7)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m7" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">NPV</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">FOR</mml:mi></mml:mrow></mml:mrow></mml:math></span></div></div></p><p>The accuracy can also be defined in terms of precision and inverse precision as follows [<a href="#ref016" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016" alt="" title="">16</a>]:<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn8"><span class="label">(8)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m8" display="block"><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mi mathvariant="italic">NPV</mml:mi></mml:mrow><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">P</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">N</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:math></span></div></div></p></section><section tabindex="0" id="sec002.6"> <h3 class="intent_subheading">2.6 Likelihood ratio</h3><p>The likelihood ratio combines both sensitivity and specificity, and it is used in diagnostic tests. In that tests, not all positive results are true positives and also the same for negative results; hence, the positive and negative results change the probability/likelihood of diseases. Likelihood ratio measures the influence of a result on the probability. <em>Positive likelihood</em> (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>+</mml:mo></mml:mrow></mml:mrow></mml:math></span>) measures how much the odds of the disease increases when a diagnostic test is positive, and it is calculated as in <a href="#eqn9" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn9" alt="" title="">Eq. (9)</a> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. Similarly, <em>Negative likelihood</em> (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>−</mml:mo></mml:mrow></mml:mrow></mml:math></span>) measures how much the odds of the disease decreases when a diagnostic test is negative, and it is calculated as in <a href="#eqn9" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn9" alt="" title="">Eq. (9)</a>. Both measures depend on the sensitivity and specificity measures; thus, they are suitable for balanced and imbalanced data [<a href="#ref006" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref006" alt="" title="">6</a>].<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn9"><span class="label">(9)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m9" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>+</mml:mo><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi></mml:mrow></mml:mfrac><mml:mo>,</mml:mo><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>−</mml:mo><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div></p><p>Both <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>+</mml:mo></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>−</mml:mo></mml:mrow></mml:mrow></mml:math></span> are combined into one measure which summarizes the performance of the test, this measure is called <em>Diagnostic odds ratio</em> (<em>DOR</em>). The <em>DOR</em> metric represents the ratio between the positive likelihood ratio to the negative likelihood ratio as in <a href="#eqn10" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn10" alt="" title="">Eq. (10)</a>. This measure is utilized for estimating the discriminative ability of the test and also for comparing between two diagnostic tests. From <a href="#eqn10" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn10" alt="" title="">Eq. (10)</a> it can be remarked that the value of <em>DOR</em> increases when (1) the <em>TP</em> and <em>TN</em> are high and (2) the <em>FP</em> and <em>FN</em> are low [<a href="#ref018" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref018" alt="" title="">18</a>].<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn10"><span class="label">(10)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m10" display="block"><mml:mrow><mml:mi mathvariant="italic">DOR</mml:mi><mml:mo>=</mml:mo></mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>+</mml:mo></mml:mrow><mml:mrow><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>−</mml:mo></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:math></span></div></div></p></section><section tabindex="0" id="sec002.7"> <h3 class="intent_subheading">2.7 Youden’s index</h3><p><em>Youden’s index</em> (<em>YI</em>) or <em>Bookmaker Informedness</em> (BM) metric is one of the well-known diagnostic tests. It evaluates the discriminative power of the test. The formula of Youden’s index combines the sensitivity and specificity as in the DOR metric, and it is defined as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">YI</mml:mi><mml:mo>=</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo>−</mml:mo><mml:mn>1</mml:mn></mml:mrow></mml:mrow></mml:math></span> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. The <em>YI</em> metric is ranged from zero when the test is poor to one which represents a perfect diagnostic test. It is also suitable with imbalanced data. One of the major disadvantages of this test is that it does not change concerning the differences between the sensitivity and specificity of the test. For example, given two tests, the sensitivity values for the first and second tests are 0.7 and 0.9, respectively, and the specificity values for the first and second tests are 0.8 and 0.6, respectively; the <em>YI</em> value for both tests is 0.5.</p></section><section tabindex="0" id="sec002.8"> <h3 class="intent_subheading">2.8 Another metrics</h3><p>There are many different metrics that can be calculated from the previous metrics. Some details about each measure are as follow:<ul class="intent_list" id="list1" list-type="bullet"><li class="intent_item"><div class="listItemContents"><p><em>Matthews correlation coefficient</em> (<em>MCC</em>): this metric was introduced by Brian W. Matthews in 1975 [<a href="#ref014" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref014" alt="" title="">14</a>], and it represents the correlation between the observed and predicted classifications, and it is calculated directly from the confusion matrix as in <a href="#eqn11" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn11" alt="" title="">Eq. (11)</a>. A coefficient of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo>+</mml:mo><mml:mn>1</mml:mn></mml:mrow></mml:mrow></mml:math></span> indicates a perfect prediction, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo>−</mml:mo><mml:mn>1</mml:mn></mml:mrow></mml:mrow></mml:math></span> represents total disagreement between prediction and true values and zero means that no better than random prediction [<a href="#ref016 ref003" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016 ref003" alt="" title="">16,3</a>]. This metric is sensitive to imbalanced data.</p></div></li></ul><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn11"><span class="label">(11)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m11" display="block"><mml:mrow><mml:mrow><mml:mtable columnalign="left"><mml:mtr columnalign="left"><mml:mtd columnalign="left"><mml:mrow><mml:mi mathvariant="italic">MCC</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>−</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mrow><mml:msqrt><mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow></mml:mrow></mml:msqrt></mml:mrow></mml:mfrac></mml:mrow></mml:mtd></mml:mtr><mml:mtr columnalign="left"><mml:mtd columnalign="left"><mml:mrow><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">N</mml:mi></mml:mfrac><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi></mml:mrow><mml:mrow><mml:msqrt><mml:mrow><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow></mml:mrow></mml:msqrt></mml:mrow></mml:mfrac></mml:mrow></mml:mtd></mml:mtr></mml:mtable></mml:mrow></mml:mrow></mml:math></span></div></div><ul class="intent_list" id="list2" list-type="bullet"><li class="intent_item"><div class="listItemContents"><p><em>Discriminant power</em> (<em>DP</em>): this measure depends on the sensitivity and specificity and it is defined as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">DP</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:msqrt><mml:mn>3</mml:mn></mml:msqrt></mml:mrow><mml:mi mathvariant="italic">π</mml:mi></mml:mfrac><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mi mathvariant="italic">log</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>+</mml:mo><mml:mi mathvariant="italic">log</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow></mml:mfrac><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. This metric evaluates how well the classification model distinguishes between positive and negative samples. Since this metric depends on the sensitivity and specificity metrics; it can be used with imbalanced data.</p></div></li><li class="intent_item"><div class="listItemContents"><p><em>F-measure:</em> this is also called <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>F</mml:mi><mml:mn>1</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span>-score, and it represents the harmonic mean of precision and recall as in <a href="#eqn12" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn12" alt="" title="">Eq. (12)</a> [<a href="#ref020" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref020" alt="" title="">20</a>]. The value of <em>F</em>-measure is ranged from zero to one, and high values of <em>F</em>-measure indicate high classification performance. This measure has another variant which is called <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>F</mml:mi><mml:mi>β</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span>-measure. This variant represents the weighted harmonic mean between precision and recall as in <a href="#eqn13" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn13" alt="" title="">Eq. (13)</a>. This metric is sensitive to changes in data distributions. Assume that the negative class samples are increased by <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mi>α</mml:mi></mml:mrow></mml:math></span> times; thus, the <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>F</mml:mi><mml:mo>−</mml:mo><mml:mtext>measure</mml:mtext></mml:mrow></mml:mrow></mml:math></span> is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>F</mml:mi><mml:mo>−</mml:mo><mml:mtext>measure</mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>2</mml:mn><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mn>2</mml:mn><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span> and hence this metric is affected by the changes in the class distribution.</p></div></li></ul><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn12"><span class="label">(12)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m12" display="block"><mml:mtable columnalign="left"><mml:mtr><mml:mtd><mml:mi>F</mml:mi><mml:mo>−</mml:mo><mml:mtext>measure</mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>2</mml:mn><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow></mml:mfrac></mml:mtd></mml:mtr><mml:mtr><mml:mtd><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>2</mml:mn><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mn>2</mml:mn><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mtd></mml:mtr></mml:mtable></mml:math></span></div></div><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn13"><span class="label">(13)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m13" display="block"><mml:mtable columnalign="left"><mml:mtr><mml:mtd><mml:msub><mml:mi mathvariant="italic">F</mml:mi><mml:mi mathvariant="italic">β</mml:mi></mml:msub><mml:mo>−</mml:mo><mml:mtext>measure</mml:mtext><mml:mo>=</mml:mo><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:msup><mml:mi mathvariant="italic">β</mml:mi><mml:mn>2</mml:mn></mml:msup></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>.</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mrow><mml:msup><mml:mi mathvariant="italic">β</mml:mi><mml:mn>2</mml:mn></mml:msup><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow></mml:mfrac></mml:mtd></mml:mtr><mml:mtr><mml:mtd><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mtext> </mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:msup><mml:mi mathvariant="italic">β</mml:mi><mml:mn>2</mml:mn></mml:msup></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:msup><mml:mi mathvariant="italic">β</mml:mi><mml:mn>2</mml:mn></mml:msup></mml:mrow><mml:mo stretchy="true">)</mml:mo></mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:msup><mml:mi mathvariant="italic">β</mml:mi><mml:mn>2</mml:mn></mml:msup><mml:mi mathvariant="italic">FN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac></mml:mtd></mml:mtr></mml:mtable></mml:math></span></div></div></p><p>Adjusted <em>F</em>-measure (<em>AGF</em>) was introduced in [<a href="#ref013" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref013" alt="" title="">13</a>]. The <em>F</em>-measures used only three of the four elements of the confusion matrix and hence two classifiers with different <em>TNR</em> values may have the same <em>F</em>-score. Therefore, the <em>AGF</em> metric is introduced to use all elements of the confusion matrix and provide more weights to samples which are correctly classified in the minority class. This metric is defined as follows:<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn14"><span class="label">(14)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m14" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">AGF</mml:mi><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:msub><mml:mi mathvariant="italic">F</mml:mi><mml:mn>2</mml:mn></mml:msub><mml:msub><mml:mrow><mml:mi mathvariant="italic">.InvF</mml:mi></mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:msqrt></mml:mrow></mml:mrow></mml:math></span></div></div>where <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>F</mml:mi><mml:mn>2</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span> is the <em>F</em>-measure where <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>β</mml:mi><mml:mo>=</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">InvF</mml:mi></mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span> is calculated by building a new confusion matrix where the class label of each sample is switched (i.e. positive samples become negative and vice versa).<ul class="intent_list" id="list3" list-type="bullet"><li class="intent_item"><div class="listItemContents"><p><em>Markedness</em> (<em>MK</em>): this is defined based on PPV and NPV metrics as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">MK</mml:mi><mml:mo>=</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">NPV</mml:mi><mml:mo>−</mml:mo><mml:mn>1</mml:mn></mml:mrow></mml:mrow></mml:math></span> [<a href="#ref016" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016" alt="" title="">16</a>]. This metric sensitive to data changes and hence it is not suitable for imbalanced data. This is because the Markedness metric depends on PPV and NPV metrics and both PPV and NPV are sensitive to changes in data distributions.</p></div></li><li class="intent_item"><div class="listItemContents"><p><em>Balanced classification rate or balanced accuracy</em> (<em>BCR</em>): this metric combines the sensitivity and specificity metrics and it is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">BCR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mn>1</mml:mn><mml:mn>2</mml:mn></mml:mfrac><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>=</mml:mo><mml:mfrac><mml:mn>1</mml:mn><mml:mn>2</mml:mn></mml:mfrac><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span>. Also, <em>Balance error rate</em> (<em>BER</em>) or <em>Half total error rate</em> (<em>HTER</em>) represents <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">BCR</mml:mi></mml:mrow></mml:mrow></mml:math></span>. Both <em>BCR</em> and <em>BER</em> metrics can be used with imbalanced datasets.</p></div></li><li class="intent_item"><div class="listItemContents"><p><em>Geometric Mean</em> (<em>GM</em>): The main goal of all classifiers is to improve the sensitivity, without sacrificing the specificity. However, the aims of sensitivity and specificity are often conflicting, which may not work well, especially when the dataset is imbalanced. Hence, the <em>Geometric Mean</em> (<em>GM</em>) metric aggregates both sensitivity and specificity measures according to <a href="#eqn15" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn15" alt="" title="">Eq. (15)</a> [<a href="#ref003" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref003" alt="" title="">3</a>]. <em>Adjusted Geometric Mean</em> (<em>AGM</em>) is proposed to obtain as much information as possible about each class [<a href="#ref011" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref011" alt="" title="">11</a>]. The <em>AGM</em> metric is defined according to <a href="#eqn16" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn16" alt="" title="">Eq. (16)</a>.</p></div></li></ul><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn15"><span class="label">(15)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m15" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">GM</mml:mi><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>×</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:msqrt><mml:mo>=</mml:mo><mml:msqrt><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:msqrt></mml:mrow></mml:mrow></mml:math></span></div></div><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn16"><span class="label">(16)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m16" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">AGM</mml:mi><mml:mo>=</mml:mo><mml:mrow><mml:mo stretchy="true">{</mml:mo><mml:mrow><mml:mtable columnalign="left"><mml:mtr columnalign="left"><mml:mtd columnalign="left"><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">GM</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo stretchy="false">(</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mtd><mml:mtd columnalign="left"><mml:mrow><mml:mtext>if </mml:mtext><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>></mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mtd></mml:mtr><mml:mtr columnalign="left"><mml:mtd columnalign="left"><mml:mtext>0</mml:mtext></mml:mtd><mml:mtd columnalign="left"><mml:mrow><mml:mtext>if </mml:mtext><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mtd></mml:mtr></mml:mtable></mml:mrow></mml:mrow></mml:mrow></mml:mrow></mml:math></span></div></div></p><p>GM metric can be used with imbalanced datasets. Lopez et al. reported that the <em>AGM</em> metric is suitable with the imbalanced data [<a href="#ref012" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref012" alt="" title="">12</a>]. However, changing the distribution of negative class has a small influence on the <em>AGM</em> metric and hence it is not suitable with the imbalanced data. This is can be proved simply by assuming that the negative class samples are increased by <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mi>α</mml:mi></mml:mrow></mml:math></span> times. Thus, the <em>AGM</em> metric is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">AGM</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">GM</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">α</mml:mi><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>; as a consequence, the <em>AGM</em> metric is slightly affected by the changes in the class distribution.<ul class="intent_list" id="list4" list-type="bullet"><li class="intent_item"><div class="listItemContents"><p><em>Optimization precision</em> (<em>OP</em>): This metric is defined as follows:</p></div></li></ul><div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn17"><span class="label">(17)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m17" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">OP</mml:mi><mml:mo>=</mml:mo><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>−</mml:mo><mml:mfrac><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo stretchy="false">|</mml:mo></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div>where the second term <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mfrac><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo stretchy="false">|</mml:mo></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:math></span> computes how balanced both class accuracies are and this metric represents the difference between the global accuracy and that term [<a href="#ref009" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref009" alt="" title="">9</a>]. High <em>OP</em> value indicates high accuracy and well-balanced class accuracies. Since the <em>OP</em> metric depends on the accuracy metric, it is not suitable for imbalanced data.<ul class="intent_list" id="list5" list-type="bullet"><li class="intent_item"><div class="listItemContents"><p><em>Jaccard</em>: This metric is also called Tanimoto similarity coefficient. Jaccard metric explicitly ignores the correct classification of negative samples as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Jaccard</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>. Jaccard metric is sensitive to changes in data distributions.</p></div></li></ul></p><p><a href="#F_j.aci.2018.08.003004" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003004" alt="Results of a multi-class classification test (our example)." title="Results of a multi-class classification test (our example).">Figure 4</a> shows the relations between different classification assessment methods. As shown, all assessment methods can be calculated from the confusion matrix. As shown, there are two classes; red class and blue class. After applying a classifier, the classifier is represented by a black circle and the samples which are inside the circle are classified as red class samples and the samples outside the circle are classified as blue class samples. Additionally, from the figure, it is clear that many assessment methods depend on the <em>TPR</em> and <em>TNR</em> metrics, and all assessment methods can be estimated from the confusion matrix.</p></section><section tabindex="0" id="sec002.9"> <h3 class="intent_subheading">2.9 Illustrative example</h3><p>In this section, two examples are introduced. These examples explain how to calculate classification metrics using two classes or multiple classes.</p><section tabindex="0" id="sec002.9.1"> <h4 class="intent_subheading">2.9.1 Binary classification example</h4><p>In this example, assume we have two classes (A and B), i.e., binary classification, and each class has 100 samples. The A class represents the positive class while the B class represents the negative class. The number of correctly classified samples in class A and B are 70 and 80, respectively. Hence, the values of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mrow></mml:math></span>, and <em>FN</em> are 70, 80, 20, and 30, respectively. The values of different classification metrics are as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>80</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mn>20</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>75</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>80</mml:mn></mml:mrow><mml:mrow><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mn>20</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>20</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>78</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">NPV</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>80</mml:mn></mml:mrow><mml:mrow><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>73</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">Err</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>25</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">BCR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mn>1</mml:mn><mml:mn>2</mml:mn></mml:mfrac><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>75</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FPR</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>2</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FNR</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>3</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">F</mml:mi><mml:mo>−</mml:mo><mml:mtext>measure</mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>20</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>74</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">OP</mml:mi><mml:mo>=</mml:mo><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>−</mml:mo><mml:mfrac><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>−</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo stretchy="false">|</mml:mo></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>75</mml:mn><mml:mo>−</mml:mo><mml:mfrac><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>−</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn><mml:mo stretchy="false">|</mml:mo></mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>683</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>+</mml:mo><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn></mml:mrow><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mn>3</mml:mn><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">LR</mml:mi><mml:mo>−</mml:mo><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn></mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>375</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">DOR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>3</mml:mn><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn></mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>375</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mn>9</mml:mn><mml:mtext>.</mml:mtext><mml:mn>33</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">YI</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn><mml:mo>−</mml:mo><mml:mn>1</mml:mn><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Jaccard</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>20</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>583</mml:mn></mml:mrow></mml:mrow></mml:math></span>.</p><p>We increased the number of samples of the B class to 1000 to show how the classification metrics are changed when using imbalanced data, and there are 800 samples from class B were correctly classified. As a consequence, the values of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mrow></mml:math></span>, and <em>FN</em> are 70, 800, 200, and 30, respectively. Consequently, only the values of accuracy, precision/PPV, NPV, error rate, Optimization precision, F-measure, and Jaccard are changed as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>800</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>800</mml:mn><mml:mo>+</mml:mo><mml:mn>200</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>79</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>200</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>26</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">NPV</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>800</mml:mn></mml:mrow><mml:mrow><mml:mn>800</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>96</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">Err</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>21</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">OP</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>79</mml:mn><mml:mo>−</mml:mo><mml:mfrac><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>−</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn><mml:mo stretchy="false">|</mml:mo></mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>723</mml:mn><mml:mo>,</mml:mo><mml:mi mathvariant="italic">F</mml:mi><mml:mo>−</mml:mo><mml:mtext>measure</mml:mtext><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>200</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>378</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Jaccard</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>200</mml:mn><mml:mo>+</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>233</mml:mn></mml:mrow></mml:mrow></mml:math></span>. This example reflects that the accuracy, precision, NPV, F-measure, and Jaccard metrics are sensitive to imbalanced data.</p></section><section tabindex="0" id="sec002.9.2"> <h4 class="intent_subheading">2.9.2 Multi-classification example</h4><p>In this example, there are three classes A, B, and C, the results of a classification test are shown in <a href="#F_j.aci.2018.08.003004" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003004" alt="Results of a multi-class classification test (our example)." title="Results of a multi-class classification test (our example).">Figure 4</a>. From the figure, the values of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>,</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span>, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mtext>TB</mml:mtext></mml:mrow><mml:mi mathvariant="italic">C</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> are 80, 70, and 90, respectively, which represent the diagonal in <a href="#F_j.aci.2018.08.003004" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003004" alt="Results of a multi-class classification test (our example)." title="Results of a multi-class classification test (our example).">Figure 4</a>. The values of false negative for each class (true class) are calculated as mentioned before by adding all errors in the column of that class. For example, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AB</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AC</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mn>5</mml:mn><mml:mo>=</mml:mo><mml:mn>20</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and similarly <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">BA</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">BC</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>=</mml:mo><mml:mn>30</mml:mn></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi mathvariant="italic">C</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">CA</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">CB</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mo>+</mml:mo><mml:mn>10</mml:mn><mml:mo>=</mml:mo><mml:mn>10</mml:mn></mml:mrow></mml:mrow></mml:math></span>. The values of false positive for each class (predicted class) are calculated as mentioned before by adding all errors in the row of that class. For example, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">BA</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">CA</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mo>=</mml:mo><mml:mn>15</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and similarly <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AB</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">CB</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mn>10</mml:mn><mml:mo>=</mml:mo><mml:mn>25</mml:mn></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">C</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">AC</mml:mi></mml:mrow></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mi mathvariant="italic">E</mml:mi><mml:mrow><mml:mi mathvariant="italic">BC</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mn>5</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>=</mml:mo><mml:mn>20</mml:mn></mml:mrow></mml:mrow></mml:math></span>. The value of true negative for the class A (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span>) can be calculated by adding all columns and rows excluding the row and column of class A; this is similar to the <em>TN</em> in the <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span> confusion matrix. Hence, the value of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>90</mml:mn><mml:mo>+</mml:mo><mml:mn>10</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>=</mml:mo><mml:mn>185</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and similarly <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mo>+</mml:mo><mml:mn>5</mml:mn><mml:mo>+</mml:mo><mml:mn>90</mml:mn><mml:mo>=</mml:mo><mml:mn>175</mml:mn></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TN</mml:mi></mml:mrow><mml:mi mathvariant="italic">C</mml:mi></mml:msub><mml:mo>=</mml:mo><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>=</mml:mo><mml:mn>180</mml:mn></mml:mrow></mml:mrow></mml:math></span>. Using <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mrow></mml:math></span>, and <em>FN</em> we can calculate all classification measures. For example, the accuracy is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>90</mml:mn></mml:mrow><mml:mrow><mml:mn>100</mml:mn><mml:mo>+</mml:mo><mml:mn>100</mml:mn><mml:mo>+</mml:mo><mml:mn>100</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mrow></mml:math></span>. The sensitivity and specificity are calculated for each class. For example, the sensitivity of A is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>80</mml:mn></mml:mrow><mml:mrow><mml:mn>80</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mn>5</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and similarly the sensitivity of B and C classes are <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mn>70</mml:mn></mml:mrow><mml:mrow><mml:mn>70</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>7</mml:mn></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mn>90</mml:mn></mml:mrow><mml:mrow><mml:mn>90</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mo>+</mml:mo><mml:mn>10</mml:mn></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>9</mml:mn></mml:mrow></mml:mrow></mml:math></span>, respectively, and the specificity values of A, B, and C are <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mn>185</mml:mn></mml:mrow><mml:mrow><mml:mn>185</mml:mn><mml:mo>+</mml:mo><mml:mn>15</mml:mn></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>93</mml:mn><mml:mo>,</mml:mo><mml:mfrac><mml:mrow><mml:mn>175</mml:mn></mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mn>175</mml:mn><mml:mo>+</mml:mo><mml:mn>25</mml:mn><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>875</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mn>180</mml:mn></mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mn>180</mml:mn><mml:mo>+</mml:mo><mml:mn>20</mml:mn><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>9</mml:mn></mml:mrow></mml:mrow></mml:math></span>, respectively.</p></section></section></section> <section tabindex="0" id="sec003"> <h2 class="intent_subheading">3. Receiver operating characteristics (ROC)</h2><p>The <em>receiver operating characteristics</em> (ROC) curve is a two-dimensional graph in which the <em>TPR</em> represents the <em>y</em>-axis and <em>FPR</em> is the <em>x</em>-axis. The ROC curve has been used to evaluate many systems such as diagnostic systems, medical decision-making systems, and machine learning systems [<a href="#ref026" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref026" alt="" title="">26</a>]. It is used to make a balance between the benefits, i.e., true positives, and costs, i.e., false positives. Any classifier that has discrete outputs such as decision trees is designed to produce only a class decision, i.e., a decision for each testing sample, and hence it generates only one confusion matrix which in turn corresponds to one point into the ROC space. However, there are many methods that were introduced for generating full ROC curve from a classifier instead of only a single point such as using class proportions [<a href="#ref026" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref026" alt="" title="">26</a>] or using some combinations of scoring and voting [<a href="#ref008" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref008" alt="" title="">8</a>]. On the other hand, in continuous output classifiers such as the Naive Bayes classifier, the output is represented by a numeric value, i.e., score, which represents the degree to which a sample belongs to a specific class. The ROC curve is generated by changing the threshold on the confidence score; hence, each threshold generates only one point in the ROC curve [<a href="#ref008" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref008" alt="" title="">8</a>].</p><p><a href="#F_j.aci.2018.08.003005" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003005" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." title="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.">Figure 5</a> shows an example of the ROC curve. As shown, there are four important points in the ROC curve. The point A, in the lower left corner <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mtext>0</mml:mtext><mml:mo>,</mml:mo><mml:mtext>0</mml:mtext><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:math></span> represents a classifier where there is no positive classification, while all negative samples are correctly classified and hence <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mrow></mml:math></span>. The point C, in the top right corner (1,1), represents a classifier where all positive samples are correctly classified, while the negative samples are misclassified. The point D in the lower right corner <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mn>1</mml:mn><mml:mo>,</mml:mo><mml:mtext>0</mml:mtext><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:math></span> represents a classifier where all positive and negative samples are misclassified. The point B in the upper left corner <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mtext>0</mml:mtext><mml:mo>,</mml:mo><mml:mn>1</mml:mn><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:math></span> represents a classifier where all positive and negative samples are correctly classified; thus, this point represents the perfect classification or the <em>Ideal operating point</em>. <a href="#F_j.aci.2018.08.003005" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003005" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." title="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.">Figure 5</a> shows the perfect classification performance. It is the green curve which rises vertically from (0,0) to (0,1) and then horizontally to (1,1). This curve reflects that the classifier perfectly ranked the positive samples relative to the negative samples.</p><p>A point in the ROC space is better than all other points that are in the southeast, i.e., the points that have lower <em>TPR</em>, higher <em>FPR</em>, or both (see <a href="#F_j.aci.2018.08.003005" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003005" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." title="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.">Figure 5</a>). Therefore, any classifier appears in the lower right triangle performs worse than the classifier appears in the upper left triangle.</p><p><a href="#F_j.aci.2018.08.003006" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003006" alt="An illustrative example to calculate the TPR and FPR when the threshold value is changed." title="An illustrative example to calculate the TPR and FPR when the threshold value is changed.">Figure 6</a> shows an example of the ROC curve. In this example, a test set consists of 20 samples from two classes; each class has ten samples, i.e., ten positive and ten negative samples. As shown in the table in <a href="#F_j.aci.2018.08.003006" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003006" alt="An illustrative example to calculate the TPR and FPR when the threshold value is changed." title="An illustrative example to calculate the TPR and FPR when the threshold value is changed.">Figure 6</a>, the initial step to plot the ROC curve is to sort the samples according to their scores. Next, the threshold value is changed from maximum to minimum to plot the ROC curve. To scan all samples, the threshold is ranged from ∞ to –∞. The samples are classified into the positive class if their scores are higher than or equal the threshold; otherwise, it is estimated as negative [<a href="#ref008" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref008" alt="" title="">8</a>]. <a href="#F_j.aci.2018.08.003007 F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003007 F_j.aci.2018.08.003008" alt="" title="">Figures 7 and 8</a> shows how changing the threshold value changes the <em>TPR</em> and <em>FPR</em>. As shown in <a href="#F_j.aci.2018.08.003006" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003006" alt="An illustrative example to calculate the TPR and FPR when the threshold value is changed." title="An illustrative example to calculate the TPR and FPR when the threshold value is changed.">Figure 6</a>, the threshold value is set at maximum (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>1</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mi>∞</mml:mi></mml:mrow></mml:mrow></mml:math></span>); hence, all samples are classified as negative samples and the values of <em>FPR</em> and <em>TPR</em> are zeros and the position of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>1</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span> is in the lower left corner (the point (0,0)). The threshold value is decreased to <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>82</mml:mn></mml:mrow></mml:mrow></mml:math></span>, and the first sample is classified correctly as a positive sample (see <a href="#F_j.aci.2018.08.003006 F_j.aci.2018.08.003007 F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003006 F_j.aci.2018.08.003007 F_j.aci.2018.08.003008" alt="" title="">Figures 6–8(a)</a>). The <em>TPR</em> increased to <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>1</mml:mn></mml:mrow></mml:mrow></mml:math></span>, while the <em>FPR</em> remains zero. As the threshold is further reduced to be <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow></mml:mrow></mml:math></span>, the <em>TPR</em> is increased to <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span> and the <em>FPR</em> remains zero. As shown in <a href="#F_j.aci.2018.08.003007" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003007" alt="An illustrative example of the ROC curve. The values of TPR and FPR of each point/threshold are calculated in Table 1." title="An illustrative example of the ROC curve. The values of TPR and FPR of each point/threshold are calculated in Table 1.">Figure 7</a>, increasing the <em>TPR</em> moves the ROC curve up while increasing the <em>FPR</em> moves the ROC curve to the right as in <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>4</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span>. The ROC curve must pass through the point (0,0) where the threshold value is ∞ (in which all samples are classified as negative samples) and the point (1,1) where the threshold is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo>−</mml:mo><mml:mi>∞</mml:mi></mml:mrow></mml:mrow></mml:math></span> (in which all samples are classified as positive samples).</p><p><a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8</a> shows graphically the performance of the classification model with different threshold values. From this figure, the following remarks can be drawn.<ul class="intent_list" id="list6" list-type="bullet"><li class="intent_item"><div class="listItemContents"><p><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>1</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span>: The value of this threshold was ∞ as shown in <a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8a</a>) and hence all samples are classified as negative samples. This means that <a href="#eqn1" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn1" alt="" title="">(1)</a> all positive samples are incorrectly classified; hence, the value of <em>TP</em> is zero, (2) all negative samples are correctly classified and hence there is no <em>FP</em> (see also <a href="#F_j.aci.2018.08.003006" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003006" alt="An illustrative example to calculate the TPR and FPR when the threshold value is changed." title="An illustrative example to calculate the TPR and FPR when the threshold value is changed.">Figure 6</a>).</p></div></li><li class="intent_item"><div class="listItemContents"><p><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>3</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span>: The threshold value decreased as shown in <a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8b</a>) and as shown there are two positive samples are correctly classified. Therefore, according to the positive class, only the positive samples which have scores more than or equal this threshold (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>3</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span>) will be correctly classified, i.e., <em>TP</em>, while the other positive samples are incorrectly classified, i.e., <em>FN.</em> In this threshold, also all negative samples are correctly classified; thus, the value of <em>FP</em> is still zero.</p></div></li><li class="intent_item"><div class="listItemContents"><p><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>8</mml:mn></mml:msub></mml:mrow></mml:mrow></mml:math></span>: As the threshold further decreased to be <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>54</mml:mn></mml:mrow></mml:mrow></mml:math></span>, the threshold line moves to the left. This means that more positive samples have the chance to be correctly classified; on the other hand, some negative samples are misclassified. As a consequence, the values of <em>TP</em> and <em>FP</em> are increased as shown in <a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8(c)</a>, and the values of <em>TN</em> and <em>FN</em> decreased.</p></div></li><li class="intent_item"><div class="listItemContents"><p><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>11</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span>: This is an important threshold value where the numbers of errors from both positive and negative classes are equal (see <a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8(d)</a>) <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>=</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>=</mml:mo><mml:mn>6</mml:mn></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>=</mml:mo><mml:mi mathvariant="italic">FN</mml:mi><mml:mo>=</mml:mo><mml:mn>4</mml:mn></mml:mrow></mml:mrow></mml:math></span>).</p></div></li><li class="intent_item"><div class="listItemContents"><p><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>14</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span>: Reducing the value of the threshold to <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>37</mml:mn></mml:mrow></mml:mrow></mml:math></span> results more correctly classified positive samples and this increases <em>TP</em> and reduces <em>FN</em> as shown in <a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8(e)</a>. On the contrary, more negative samples are misclassified and this increases <em>FP</em> and reduces <em>TN.</em></p></div></li><li class="intent_item"><div class="listItemContents"><p><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>20</mml:mn></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span>: As shown in <a href="#F_j.aci.2018.08.003008" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003008" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." title="A visualization of how changing the threshold changes the TP,TN,FP, and FN values.">Figure 8(f)</a>, decreasing the threshold value hides the <em>FN</em> area. This is because all positive samples are correctly classified. Also, from the figure, it is clear that the <em>FP</em> area is much larger than the area of <em>TN.</em> This is because 90% of the negative samples are incorrectly classified, and only 10% of negative samples are correctly classified.</p></div></li></ul></p><p>From <a href="#F_j.aci.2018.08.003007" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003007" alt="An illustrative example of the ROC curve. The values of TPR and FPR of each point/threshold are calculated in Table 1." title="An illustrative example of the ROC curve. The values of TPR and FPR of each point/threshold are calculated in Table 1.">Figure 7</a> it is clear that the ROC curve is a step function. This is because we only used 20 samples (a finite set of samples) in our example and a true curve can be obtained when the number of samples increased. The figure also shows that the best accuracy (70%) (see <a href="#tbl1" class="text-link scroll_to Link intent_table_link Link__table" data-target="tbl1" alt="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value." title="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value.">Table 1</a>) is obtained at (0.1,0.5) when the threshold value was ≥ 0.6, rather than at ≥ 0.5 as we might expect with a balanced data. This means that the given learning model identifies positive samples better than negative samples. Since the ROC curve depends mainly on changing the threshold value, comparing classifiers with different score ranges will be meaningless. For example, assume we have two classifiers, the first generates scores in the range [0,1] and the other generates scores in the range [−1,+1] and hence we cannot compare these classifiers using the ROC curve.</p><p>The steps of generating ROC curve are summarized in Algorithm 1. The algorithm requires <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>O</mml:mi><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mi>nlogn</mml:mi><mml:mo stretchy="true">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> for sorting samples, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>O</mml:mi><mml:mrow><mml:mo stretchy="true">(</mml:mo><mml:mi>n</mml:mi><mml:mo stretchy="true">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> for scanning them; resulting in <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>O</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mi>nlogn</mml:mi><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> total complexity, where <em>n</em> is the number of samples. As shown, the two main steps to generate ROC points are (1) sorting samples according to their scores and (2) changing the threshold value from maximum to minimum to process one sample at a time and update the values of <em>TP</em> and <em>FP</em> in each time. The algorithm shows that the <em>TP</em> and the <em>FP</em> start at zero. The algorithm scans all samples and the value of <em>TP</em> is increased for each positive sample while the value of <em>FP</em> is increased for each negative sample. Next, the values of <em>TPR</em> and <em>FPR</em> are calculated and pushed into the ROC stack (see step 6). When the threshold becomes very low (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>threshold</mml:mi><mml:mo>→</mml:mo><mml:mo>−</mml:mo><mml:mi>∞</mml:mi></mml:mrow></mml:mrow></mml:math></span>), all samples are classified as positive samples and hence the values of both <em>TPR</em> and <em>FPR</em> are one.</p><p>Steps 5–8 handle sequences of equally scored samples. Assume we have a test set which consists of <em>P</em> positive samples and <em>N</em> negative samples. In this test set, assume we have <em>p</em> positive samples and <em>n</em> negative samples with the same score value. There are two extreme cases. In the first case which is the optimistic case, all positive samples end up at the beginning of the sequence, and this case represents the upper L segment of the rectangle in <a href="#F_j.aci.2018.08.003005" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003005" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." title="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.">Figure 5</a>. In the second case, i.e., pessimistic case, all the negative samples end up at the beginning of the sequence, and this case represents the lower L segment of the rectangle in <a href="#F_j.aci.2018.08.003005" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003005" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." title="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.">Figure 5</a>. The ROC curve represents the expected performance which is the average of the two cases, and it represents the diagonal of the rectangle in <a href="#F_j.aci.2018.08.003005" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003005" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." title="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.">Figure 5</a>. The size of this rectangle is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">pn</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">PN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>, and the number of errors in both optimistic and pessimistic cases can be calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">pn</mml:mi></mml:mrow><mml:mrow><mml:mn>2</mml:mn><mml:mi mathvariant="italic">PN</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>.<img class="intent_image b-lazy" src="/insight/static/img/emerald-loading-wide-xl.gif" data-src="/insight/proxy/img?link=/resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003_fx001.tif" alt=""/></p><p>In multi-class classification problems, plotting ROC becomes much more complex than in binary classification problems. One of the well-known methods to handle this problem is to produce one ROC curve for each class. For plotting ROC of the class <em>i</em> (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>c</mml:mi><mml:mi>i</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span>), the samples from <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>c</mml:mi><mml:mi>i</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> represent positive samples and all the other samples are negative samples.</p><p>ROC curves are robust against any changes to class distributions. Hence, if the ratio of positive to negative samples changes in a test set, the ROC curve will not change. In other words, ROC curves are insensitive with the imbalanced data. This is because ROC depends on <em>TPR</em> and <em>FPR</em>, and each of them is a columnar ratio<a href="#fn003" class="text-link scroll_to Link intent_footnote_link Link__footnote" data-target="fn003" alt="" title=""><sup>3</sup></a>.</p><p>The following example compares between the ROC using balanced and imbalanced data. Assume the data is balanced and it consists of two classes each has 1000 samples. The point (0.2,0.5) on the ROC curve means that the classifier obtained 50% sensitivity (500 positive samples are correctly classified from 1000 positive samples) and 80% specificity (800 negative samples are correctly classified from 1000 negative samples). If the class distribution changed to be imbalanced and the first and second classes have 1000 and 10,000 samples, respectively. Hence, the same point (0.2, 0.5) means that the classifier obtained 50% sensitivity (500 positive samples are correctly classified from 1000 positive samples) and 80% specificity (8000 negative samples are correctly classified from 1000 negative samples). The AUC<a href="#fn004" class="text-link scroll_to Link intent_footnote_link Link__footnote" data-target="fn004" alt="" title=""><sup>4</sup></a> score for both cases are the same while the other metrics which are sensitive to the imbalanced data will be changed. For example, the accuracy rates of the classifier using the balanced and imbalanced data are 65 and 77.3%, respectively, and the precision values will be <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>71</mml:mn></mml:mrow></mml:mrow></mml:math></span> and 0.20, respectively. These results reflect how the precision and accuracy metrics are sensitive to the imbalanced data as mentioned in <a href="#sec002.1" class="text-link scroll_to Link intent_link" data-target="sec002.1" alt="" title="">Section 2.1</a>.</p><p>It is worth mentioning that the comparison between different classifiers using ROC is valid only when (1) there is only single dataset, (2) there are multiple datasets with the same data size and the same positive:negative ratio.</p></section> <section tabindex="0" id="sec004"> <h2 class="intent_subheading">4. Area under the ROC curve (AUC)</h2><p>Comparing different classifiers in the ROC curve is not easy. This is because there is no scalar value represents the expected performance. Therefore, the Area under the ROC curve (AUC) metric is used to calculate the area under the ROC curve. The AUC score is always bounded between zero and one, and there is no realistic classifier has an AUC lower than 0.5 [<a href="#ref004 ref015" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref004 ref015" alt="" title="">4,15</a>].</p><p><a href="#F_j.aci.2018.08.003009" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003009" alt="An illustrative example of the AUC metric." title="An illustrative example of the AUC metric.">Figure 9</a> shows the AUC value of two classifiers, A and B. As shown, the AUC of B classifier is greater than A; hence, it achieves better performance. Moreover, the gray shaded area is common in both classifiers, while the red shaded area represents the area where the B classifier outperforms the A classifier. It is possible for a lower AUC classifier to outperform a higher AUC classifier in a specific region. For example, in <a href="#F_j.aci.2018.08.003009" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003009" alt="An illustrative example of the AUC metric." title="An illustrative example of the AUC metric.">Figure 9</a>, the classifier B outperforms A except at <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi><mml:mo>></mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>6</mml:mn></mml:mrow></mml:mrow></mml:math></span> where A has a slight difference (blue shaded area). However, two classifiers with two different ROC curves may have the same AUC score.</p><p>The AUC value is calculated as in Algorithm 2. As shown, the steps in Algorithm 2 represent a slight modification from Algorithm 1. In other words, instead of generating ROC points in Algorithm 1, Algorithm 2 adds areas of trapezoids<a href="#fn005" class="text-link scroll_to Link intent_footnote_link Link__footnote" data-target="fn005" alt="" title=""><sup>5</sup></a> of the ROC curve [<a href="#ref004" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref004" alt="" title="">4</a>]. As shown in Algorithm 2 the AUC score can be calculated by adding the areas of trapezoids of the AUC measure. <a href="#F_j.aci.2018.08.003009" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003009" alt="An illustrative example of the AUC metric." title="An illustrative example of the AUC metric.">Figure 9</a> shows an example of one trapezoid; the base of this trapezoid is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi></mml:mrow><mml:mn>2</mml:mn></mml:msub><mml:mo>−</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi></mml:mrow><mml:mn>1</mml:mn></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:math></span>, and the height of the trapezoid is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mn>1</mml:mn></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mn>2</mml:mn></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>/</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span>; hence, the total area of this trapezoid is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">A</mml:mi><mml:mo>=</mml:mo><mml:mtext>Base</mml:mtext><mml:mo>×</mml:mo><mml:mtext>Height</mml:mtext><mml:mo>=</mml:mo><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi></mml:mrow><mml:mn>2</mml:mn></mml:msub><mml:mo>−</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FPR</mml:mi></mml:mrow><mml:mn>1</mml:mn></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>×</mml:mo><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mn>1</mml:mn></mml:msub><mml:mo>+</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi></mml:mrow><mml:mn>2</mml:mn></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mo>/</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span>.<img class="intent_image b-lazy" src="/insight/static/img/emerald-loading-wide-xl.gif" data-src="/insight/proxy/img?link=/resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003_fx002.tif" alt=""/></p><p>The AUC can be also calculated under the PR curve using the trapezoidal rule as in the ROC curve, and the AUC score of the perfect classifier in PR curves is one as in ROC curves.</p><p>In multi-class classification problems, Provost and Domingos calculated the total AUC of all classes by generating a ROC curve for each class and calculate the AUC value for each ROC curve [<a href="#ref010" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref010" alt="" title="">10</a>]. The total AUC (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">AUC</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">total</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mrow></mml:math></span>) is the summation of all AUC scores weighted by the prior probability of each class as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">AUC</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">total</mml:mi></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mstyle displaystyle="true"><mml:msub><mml:mo>∑</mml:mo><mml:mrow><mml:msub><mml:mi>C</mml:mi><mml:mi>i</mml:mi></mml:msub><mml:mo>∈</mml:mo><mml:mi>C</mml:mi></mml:mrow></mml:msub><mml:mrow><mml:mi mathvariant="italic">AUC</mml:mi></mml:mrow></mml:mstyle><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mi mathvariant="italic">c</mml:mi><mml:mi mathvariant="italic">i</mml:mi></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow><mml:mtext>.</mml:mtext><mml:mi mathvariant="italic">p</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mi mathvariant="italic">c</mml:mi><mml:mi mathvariant="italic">i</mml:mi></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span>, where <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">AUC</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mi>c</mml:mi><mml:mi>i</mml:mi></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> is the AUC under the ROC curve of the class <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>c</mml:mi><mml:mi>i</mml:mi></mml:msub><mml:mo>,</mml:mo><mml:mi>C</mml:mi></mml:mrow></mml:mrow></mml:math></span> is a set of classes, and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>p</mml:mi><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:msub><mml:mi>c</mml:mi><mml:mi>i</mml:mi></mml:msub><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:mrow></mml:mrow></mml:math></span> is the prior probability of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>c</mml:mi><mml:mi>i</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> [<a href="#ref010" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref010" alt="" title="">10</a>]. This method of calculating the AUC score is simple and fast but it is sensitive to class distributions and error costs.</p></section> <section tabindex="0" id="sec005"> <h2 class="intent_subheading">5. Precision-Recall (PR) curve</h2><p>Precision and recall metrics are widely used for evaluating the classification performance. The Precision-Recall (PR) curve has the same concept of the ROC curve, and it can be generated by changing the threshold as in ROC. However, the ROC curve shows the relation between sensitivity/recall (<em>TPR</em>) and 1-specificity (<em>FPR</em>) while the PR curve shows the relationship between recall and precision. Thus, in the PR curve, the <em>x</em>-axis is the recall and the <em>y</em>-axis is the precision, i.e., the <em>x</em>-axis of ROC curve is the <em>y</em>-axis of PR curve [<a href="#ref008" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref008" alt="" title="">8</a>]. Hence, in the PR curve, there is no need for the <em>TN</em> value.</p><p>In the PR curve, the precision value for the first point is undefined because the number of positive predictions is zero, i.e., <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mrow></mml:math></span>. This problem can be solved by estimating the first point in the PR curve from the second point. There are two cases for estimating the first point depending on the value of <em>TP</em> of the second point.<ul class="intent_list simple" id="list7" list-type="simple"><li class="intent_item"><span class="listItemLabel">1.</span><div class="listItemContents"><p>The number of true positives of the second point is zero: In this case, since the second point is (0,0), the first point is also (0,0).</p></div></li><li class="intent_item"><span class="listItemLabel">2.</span><div class="listItemContents"><p>The number of true positives of the second point is not zero: this is similar to our example where the second point is (0.1, 1.0). The first point can be estimated by drawing a horizontal line from the second point to the <em>y</em>-axis. Thus, the first point is estimated as (0.0, 1.0).</p></div></li></ul></p><p>As shown in <a href="#F_j.aci.2018.08.003010" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003010" alt="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1." title="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1.">Figure 10</a>, the PR curve is often zigzag curve; hence, PR curves tend to cross each other much more frequently than ROC curves. In the PR curve, a curve above the other has a better classification performance. The perfect classification performance in the PR curve is represented in <a href="#F_j.aci.2018.08.003010" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003010" alt="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1." title="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1.">Figure 10</a> by a green curve. As shown, this curve starts from the (0,1) horizontally to (1,1) and then vertically to (1,0), where (0,1) represents a classifier that achieves 100% precision and 0% recall, (1,1) represents a classifier that obtains 100% precision and sensitivity and this is the ideal point in the PR curve, and (1,0) indicates the classifier obtains 100% sensitivity and 0% precision. Hence, we can say that the closer the PR curve is to the upper right corner, the better the classification performance is. Since the PR curve depends only on the precision and recall measures, it ignores the performance of correctly handling negative examples (<em>TN</em>) [<a href="#ref016" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref016" alt="" title="">16</a>].</p><p><a href="#eqn18" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn18" alt="" title="">Eq. (18)</a> indicates the nonlinear interpolation of the PR curve that was introduced by Davis and Goadrich [<a href="#ref005" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref005" alt="" title="">5</a>].<div class="scroll-x"><div class="intent_formula scroll-x-content disp-formula" id="eqn18"><span class="label">(18)</span><span class="alternatives"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML" id="m18" display="block"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">y</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>+</mml:mo><mml:mi mathvariant="italic">x</mml:mi></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>+</mml:mo><mml:mi mathvariant="italic">x</mml:mi><mml:mo>+</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>−</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>−</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow></mml:mfrac><mml:mtext>.</mml:mtext><mml:mi mathvariant="italic">x</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span></div></div>where <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> represent the true positives of the first and second points, respectively, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub></mml:mrow></mml:mrow></mml:math></span> represent the false positives of the first and second points, respectively, <em>y</em> is the precision of the new point, and <em>x</em> is the recall of the new point. The value of <em>x</em> can be any value between zero and <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>−</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo stretchy="false">|</mml:mo></mml:mrow></mml:mrow></mml:math></span>. A smooth curve can be obtained by calculating many intermediate points between two points A and B. In our example in <a href="#F_j.aci.2018.08.003010" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003010" alt="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1." title="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1.">Figure 10</a>, assume the first point is the fifth point and the second point is the sixth point (see <a href="#tbl1" class="text-link scroll_to Link intent_table_link Link__table" data-target="tbl1" alt="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value." title="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value.">Table 1</a>). From <a href="#tbl1" class="text-link scroll_to Link intent_table_link Link__table" data-target="tbl1" alt="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value." title="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value.">Table 1</a>, the point A is (0.3,0.75) and the point B is (0.4,0.8). The value of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mo stretchy="false">|</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">B</mml:mi></mml:msub><mml:mo>−</mml:mo><mml:msub><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">A</mml:mi></mml:msub><mml:mo stretchy="false">|</mml:mo><mml:mo>=</mml:mo><mml:mo stretchy="false">|</mml:mo><mml:mn>4</mml:mn><mml:mo>−</mml:mo><mml:mn>3</mml:mn><mml:mo stretchy="false">|</mml:mo><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow></mml:mrow></mml:math></span> and hence the value of <em>x</em> can be any value between zero and one. Let <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>x</mml:mi><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn></mml:mrow></mml:mrow></mml:math></span>, which is the middle point between A and B and hence the recall for the new point is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>3</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>4</mml:mn></mml:mrow><mml:mn>2</mml:mn></mml:mfrac><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>35</mml:mn></mml:mrow></mml:mrow></mml:math></span>. The precision of the new point is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi>y</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>3</mml:mn><mml:mo>+</mml:mo><mml:mi>x</mml:mi></mml:mrow><mml:mrow><mml:mn>3</mml:mn><mml:mo>+</mml:mo><mml:mi>x</mml:mi><mml:mo>+</mml:mo><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:mfrac><mml:mrow><mml:mn>1</mml:mn><mml:mo>−</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mn>4</mml:mn><mml:mo>−</mml:mo><mml:mn>3</mml:mn></mml:mrow></mml:mfrac><mml:mo>×</mml:mo><mml:mi>x</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>3</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn></mml:mrow><mml:mrow><mml:mn>3</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>5</mml:mn><mml:mo>+</mml:mo><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext></mml:mrow></mml:mfrac><mml:mo>≈</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>778</mml:mn></mml:mrow></mml:mrow></mml:math></span>, where the new point using the linear interpolation is (<span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>3</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>4</mml:mn></mml:mrow><mml:mn>2</mml:mn></mml:mfrac><mml:mo>,</mml:mo><mml:mfrac><mml:mrow><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>75</mml:mn><mml:mo>+</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>8</mml:mn></mml:mrow><mml:mn>2</mml:mn></mml:mfrac><mml:mo stretchy="false">)</mml:mo><mml:mo>=</mml:mo><mml:mo stretchy="false">(</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>35</mml:mn><mml:mo>,</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>775</mml:mn></mml:mrow></mml:mrow></mml:math></span>). In our example, for simplicity, we used the linear interpolation.</p><p>The end point in the PR curve is calculated as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mo stretchy="false">(</mml:mo><mml:mrow><mml:mn>1</mml:mn><mml:mo>,</mml:mo><mml:mtext> </mml:mtext><mml:mfrac><mml:mi>P</mml:mi><mml:mrow><mml:mi>P</mml:mi><mml:mo>+</mml:mo><mml:mi>N</mml:mi></mml:mrow></mml:mfrac></mml:mrow><mml:mo stretchy="false">)</mml:mo></mml:mrow></mml:math></span>. This is because (1) the recall increases by increasing the threshold value and at the end point the recall reaches to the maximum recall, (2) increasing the threshold value increases both <em>TP</em> and <em>FP</em>. Therefore, if the data are balanced, the precision of the end point is <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mi>P</mml:mi><mml:mrow><mml:mi>P</mml:mi><mml:mo>+</mml:mo><mml:mi>N</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mn>1</mml:mn><mml:mn>2</mml:mn></mml:mfrac></mml:mrow></mml:mrow></mml:math></span>. The horizontal line which passes through <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mfrac><mml:mi>P</mml:mi><mml:mrow><mml:mi>P</mml:mi><mml:mo>+</mml:mo><mml:mi>N</mml:mi></mml:mrow></mml:mfrac></mml:mrow></mml:mrow></mml:math></span> represents a classifier with the random performance level. This line separates the area of the PR curve into (1) the area above the line and this is the area of good performance and (2) the area below the line and this is the area of poor performance (see <a href="#F_j.aci.2018.08.003010" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003010" alt="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1." title="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1.">Figure 10</a>). Thus, the ratio of positives and negatives defines the baseline. Hence, changing the ratio between the positive and negative classes changes that line and hence changes the classification performance.</p><p>As indicated in <a href="#eqn6" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn6" alt="" title="">Eq. (6)</a>, according to the precision metric, lowering the threshold value increases the <em>TP</em> or <em>FP</em>. Increasing <em>TP</em> increases the precision while increasing the <em>FP</em> decreases the precision. Hence, lowering the threshold value fluctuates the precision. On the other hand, as indicated in <a href="#eqn2" class="text-link scroll_to Link intent_display_formula_link Link__display-formula" data-target="eqn2" alt="" title="">Eq. (2)</a>, lowering the threshold may leave the recall value unchanged or increase it. Due to the precision axis in the PR curve; hence, the PR curve is sensitive to the imbalanced data. In other words, the PR curves and their AUC values are different between balanced and imbalanced data.</p></section> <section tabindex="0" id="sec006"> <h2 class="intent_subheading">6. Biometrics measures</h2><p>Biometrics matching is slightly different than the other classification problems and hence it is sometimes called two-instance problem. In this problem, instead of classifying one sample into one of <em>c</em> groups or classes, biometric determines if the two samples are in the same group. This can be achieved by identifying an unknown sample by matching it with all the other known samples. This step generates a score or similarity distance between the unknown sample and the other samples. The model assigns the unknown sample to the person which has the most similar score. If this level of similarity is not reached, the sample is rejected. In other words, if the similarity score exceeds a pre-defined threshold; hence, the corresponding sample is said to be matched; otherwise, the sample is not matched. Theoretically, scores of clients (persons known by the biometric system) should always be higher than the scores of imposters (persons who are not known by the system). In biometric systems, a single threshold separates the two groups of scores; thus, it can be utilized for differentiating between clients and imposters. In real applications, for many reasons sometimes imposter samples generate scores higher than the scores of some client samples. Accordingly, it is a fact that however the classification threshold is perfectly chosen, some classification errors occur. For example, given a high threshold; hence, the imposters’ scores will not exceed this limit. As a result, no imposters are incorrectly accepted by the model. On the contrary, some clients are falsely rejected (see <a href="#F_j.aci.2018.08.003011" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003011" alt="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER." title="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER.">Figure 11</a> (top panel)). In opposition to this, lowering the threshold value accepts all clients and also some imposters are falsely accepted.</p><p>Two of the most commonly used measures in biometrics are the <em>False acceptance rate</em> (<em>FAR</em>) and <em>False rejection/recognition rate</em> (<em>FRR</em>). The <em>FAR</em> is also called <em>false match rate</em> (<em>FMR</em>) and it is the ratio between the number of false acceptance to the total number of imposters attempts. Hence, it measures the likelihood that the biometric model will incorrectly accept an access by an imposter or an unauthorized user. Hence, to prevent imposter samples from being easily correctly identified by the model, the similarity score has to exceed a certain level (see <a href="#F_j.aci.2018.08.003011" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003011" alt="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER." title="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER.">Figure 11</a>) [<a href="#ref002" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref002" alt="" title="">2</a>]. The <em>FRR</em> or <em>false non-match rate</em> (<em>F NMR</em>) measures the likelihood that the biometric model will incorrectly reject a client, and it represents the ratio between the number of false recognitions to the total number of clients’ attempts [<a href="#ref002" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref002" alt="" title="">2</a>]. For example, if <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>FAR</mml:mtext><mml:mo>=</mml:mo><mml:mn>10</mml:mn></mml:mrow></mml:mrow></mml:math></span>% this means that for one hundred attempts to access the system by imposters, only ten will be succeeded and hence increasing <em>FAR</em> decreases the accuracy of the model. On the other hand, with <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mtext>FRR</mml:mtext><mml:mo>=</mml:mo><mml:mn>10</mml:mn></mml:mrow></mml:mrow></mml:math></span>%, ten authorized persons will be rejected from 100 attempts and hence reducing <em>FRR</em> will help to avoid a high number of trails of authorized clients. As a consequence, <em>FAR</em> and <em>FRR</em> in biometrics are similar to false positive rate (<em>FPR</em>) and false negative rate (<em>FNR</em>), respectively (see <a href="#sec002.4" class="text-link scroll_to Link intent_link" data-target="sec002.4" alt="" title="">Section 2.4</a>). Equal error rate (<em>EER</em>) measure solves the problem of selecting a threshold value partially, and it represents the failure rate when the values of <em>FMR</em> and <em>F NMR</em> are equal. <a href="#F_j.aci.2018.08.003011" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003011" alt="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER." title="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER.">Figure 11</a> shows the <em>FAR</em> and <em>FRR</em> curves and also the <em>EER</em> measure.</p><p>Detection Error Trade-off (DET) curve is used for evaluating biometric models. In this curve, as in the ROC and PR curves, the threshold value is changed and the values of <em>FAR</em> and <em>FRR</em> are calculated at each threshold. Hence, this curve shows the relation between <em>FAR</em> and <em>FRR</em>. <a href="#F_j.aci.2018.08.003012" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003012" alt="An illustrative example of the DET curve. The values of FRR and FAR of each point/threshold are calculated in Table 1." title="An illustrative example of the DET curve. The values of FRR and FAR of each point/threshold are calculated in Table 1.">Figure 12</a> shows an example of the DET curve. As shown, as in the ROC curve, the DET curve is plotted by changing the threshold on the confidence score; thus, each threshold generates only one point in the DET curve. The ideal point in this curve is the origin point where the values of both <em>FRR</em> and <em>FAR</em> are zeros and hence the perfect classification performance in the DET curve is represented in <a href="#F_j.aci.2018.08.003012" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003012" alt="An illustrative example of the DET curve. The values of FRR and FAR of each point/threshold are calculated in Table 1." title="An illustrative example of the DET curve. The values of FRR and FAR of each point/threshold are calculated in Table 1.">Figure 12</a> by a green curve. As shown, this curve starts from the point (0,1) vertically to (0,0) and then horizontally to (1,0), where (1) the point (0,1) represents a classifier that achieves 100% <em>FAR</em> and 0% <em>FRR</em>, (2) the point (0,0) represents a classifier that obtains 0% <em>FAR</em> and <em>FRR</em>, and (3) the point (1,0) represents a classifier that indicates 0% <em>FAR</em> and 100% <em>FRR</em>. Thus, we can say that the closer a DET curve is to the lower left corner, the better the classification performance is.</p></section> <section tabindex="0" id="sec007"> <h2 class="intent_subheading">7. Experimental results</h2><p>In this section, an experiment was conducted to evaluate the classification performance using different assessment methods. In this experiment, we used Iris dataset which is one of the standard classification datasets and it is obtained from the University of California at Irvin (UCI) Machine Learning Repository [<a href="#ref001" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref001" alt="" title="">1</a>]. This dataset has three classes, each class has 50 samples, and each sample is represented by four features. We used (1) the Principal component analysis (PCA) [<a href="#ref023" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref023" alt="" title="">23</a>] for reducing the features to two features and (2) Support vector machine (SVM)<a href="#fn006" class="text-link scroll_to Link intent_footnote_link Link__footnote" data-target="fn006" alt="" title=""><sup>6</sup></a> for classification.</p><p>In our experiment, we used different assessment methods for evaluating the learning model. <a href="#F_j.aci.2018.08.003013" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003013" alt="Results of our experiment. (a) ROC curve, (b) Precision-Recall curve." title="Results of our experiment. (a) ROC curve, (b) Precision-Recall curve.">Figure 13</a> shows the ROC and Precision-Recall curves. As shown, there are three curves, one curve for each class and as shown, the first class obtained results better than the other two classes. <a href="#F_j.aci.2018.08.003014" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003014" alt="Confusion matrices of the three classes in our experiments." title="Confusion matrices of the three classes in our experiments.">Figure 14</a> shows the confusion matrix for each class. From these confusion matrices we can calculate different metrics as mentioned before (see <a href="#F_j.aci.2018.08.003003" class="text-link scroll_to Link intent_figure_link Link__fig" data-target="F_j.aci.2018.08.003003" alt="Visualization of different metrics and the relations between these metrics. Given two classes, red class and blue class. The black circle represents a classifier that classifies the sample inside the circle as red samples (belong to the red class) and the samples outside the circle as blue samples (belong to the blue class). Green regions indicate the correctly classified regions and the red regions indicate the misclassified regions. (For interpretation of the references to colour in this figure legend, the reader is referred to the web version of this article)." title="Visualization of different metrics and the relations between these metrics. Given two classes, red class and blue class. The black circle represents a classifier that classifies the sample inside the circle as red samples (belong to the red class) and the samples outside the circle as blue samples (belong to the blue class). Green regions indicate the correctly classified regions and the red regions indicate the misclassified regions. (For interpretation of the references to colour in this figure legend, the reader is referred to the web version of this article).">Figure 3</a>). For example, the results of the first class were as follows, <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">Acc</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TNR</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">PPV</mml:mi></mml:mrow></mml:mrow></mml:math></span>, and <em>NPV</em> were 99.33, 100, 98.0, 99.01, 100, respectively. Similarly, the results of the other two classes can be calculated.</p></section> <section tabindex="0" id="sec008"> <h2 class="intent_subheading">8. Conclusions</h2><p>In this paper, the definition, mathematics, and visualizations of the most well-known classification assessment methods were presented and explained. The paper aimed to give a detailed overview of the classification assessment measures. Moreover, based on the confusion matrix, different measures are introduced with detailed explanations. The relations between these measures and the robustness of each of them against imbalanced data are also introduced. Additionally, an illustrative numerical example was used for explaining how to calculate different classification measures with binary and multi-class problems and also to show the robustness of different measures against the imbalanced data. Graphical measures such as ROC, PR, and DET curves are also presented with illustrative examples and visualizations. Finally, various classification measures for evaluating biometric models are also presented.</p></section> </section> <section class="intent_figures Figures mt-4 "> <h2>Figures</h2> <div class="row pt-3"> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003001"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003001.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003001.tif&variant=thumbnail" alt="An illustrative example of the 2×2 confusion matrix. There are two true classes P and N. The output of the predicted class is true or false." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003001" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003001.tif&variant=thumbnail" alt="An illustrative example of the 2×2 confusion matrix. There are two true classes P and N. The output of the predicted class is true or false." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 1</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example of the <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mn>2</mml:mn><mml:mo>×</mml:mo><mml:mn>2</mml:mn></mml:mrow></mml:mrow></mml:math></span> confusion matrix. There are two true classes <em>P</em> and <em>N</em>. The output of the predicted class is true or false.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003002"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003002.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003002.tif&variant=thumbnail" alt="An illustrative example of the confusion matrix for a multi-class classification test." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003002" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003002.tif&variant=thumbnail" alt="An illustrative example of the confusion matrix for a multi-class classification test." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 2</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example of the confusion matrix for a multi-class classification test.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003003"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003003.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003003.tif&variant=thumbnail" alt="Visualization of different metrics and the relations between these metrics. Given two classes, red class and blue class. The black circle represents a classifier that classifies the sample inside the circle as red samples (belong to the red class) and the samples outside the circle as blue samples (belong to the blue class). Green regions indicate the correctly classified regions and the red regions indicate the misclassified regions. (For interpretation of the references to colour in this figure legend, the reader is referred to the web version of this article)." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003003" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003003.tif&variant=thumbnail" alt="Visualization of different metrics and the relations between these metrics. Given two classes, red class and blue class. The black circle represents a classifier that classifies the sample inside the circle as red samples (belong to the red class) and the samples outside the circle as blue samples (belong to the blue class). Green regions indicate the correctly classified regions and the red regions indicate the misclassified regions. (For interpretation of the references to colour in this figure legend, the reader is referred to the web version of this article)." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 3</h3> <p class="intent_caption card-text Figure__text"><p>Visualization of different metrics and the relations between these metrics. Given two classes, red class and blue class. The black circle represents a classifier that classifies the sample inside the circle as red samples (belong to the red class) and the samples outside the circle as blue samples (belong to the blue class). Green regions indicate the correctly classified regions and the red regions indicate the misclassified regions. (For interpretation of the references to colour in this figure legend, the reader is referred to the web version of this article).</p></p> </div> </div> </div> </div> <div class="row pt-3"> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003004"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003004.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003004.tif&variant=thumbnail" alt="Results of a multi-class classification test (our example)." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003004" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003004.tif&variant=thumbnail" alt="Results of a multi-class classification test (our example)." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 4</h3> <p class="intent_caption card-text Figure__text"><p>Results of a multi-class classification test (our example).</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003005"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003005.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003005.tif&variant=thumbnail" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003005" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003005.tif&variant=thumbnail" alt="A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 5</h3> <p class="intent_caption card-text Figure__text"><p>A basic ROC curve showing important points, and the optimistic, pessimistic and expected ROC segments for equally scored samples.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003006"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003006.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003006.tif&variant=thumbnail" alt="An illustrative example to calculate the TPR and FPR when the threshold value is changed." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003006" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003006.tif&variant=thumbnail" alt="An illustrative example to calculate the TPR and FPR when the threshold value is changed." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 6</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example to calculate the <em>TPR</em> and <em>FPR</em> when the threshold value is changed.</p></p> </div> </div> </div> </div> <div class="row pt-3"> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003007"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003007.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003007.tif&variant=thumbnail" alt="An illustrative example of the ROC curve. The values of TPR and FPR of each point/threshold are calculated in Table 1." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003007" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003007.tif&variant=thumbnail" alt="An illustrative example of the ROC curve. The values of TPR and FPR of each point/threshold are calculated in Table 1." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 7</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example of the ROC curve. The values of <em>TPR</em> and <em>FPR</em> of each point/threshold are calculated in <a href="#tbl1" class="text-link scroll_to Link intent_table_link Link__table" data-target="tbl1" alt="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value." title="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value.">Table 1</a>.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003008"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003008.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003008.tif&variant=thumbnail" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003008" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003008.tif&variant=thumbnail" alt="A visualization of how changing the threshold changes the TP,TN,FP, and FN values." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 8</h3> <p class="intent_caption card-text Figure__text"><p>A visualization of how changing the threshold changes the <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FP</mml:mi></mml:mrow></mml:mrow></mml:math></span>, and <em>FN</em> values.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003009"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003009.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003009.tif&variant=thumbnail" alt="An illustrative example of the AUC metric." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003009" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003009.tif&variant=thumbnail" alt="An illustrative example of the AUC metric." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 9</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example of the AUC metric.</p></p> </div> </div> </div> </div> <div class="row pt-3"> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003010"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003010.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003010.tif&variant=thumbnail" alt="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003010" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003010.tif&variant=thumbnail" alt="An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in Table 1." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 10</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example of the PR curve. The values of precision and recall of each point/threshold are calculated in <a href="#tbl1" class="text-link scroll_to Link intent_table_link Link__table" data-target="tbl1" alt="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value." title="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value.">Table 1</a>.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003011"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003011.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003011.tif&variant=thumbnail" alt="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003011" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003011.tif&variant=thumbnail" alt="Illustrative example to test the influence of changing the threshold value on the values of FAR,FRR, and EER." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 11</h3> <p class="intent_caption card-text Figure__text"><p>Illustrative example to test the influence of changing the threshold value on the values of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">FAR</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FRR</mml:mi></mml:mrow></mml:mrow></mml:math></span>, and EER.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003012"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003012.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003012.tif&variant=thumbnail" alt="An illustrative example of the DET curve. The values of FRR and FAR of each point/threshold are calculated in Table 1." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003012" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003012.tif&variant=thumbnail" alt="An illustrative example of the DET curve. The values of FRR and FAR of each point/threshold are calculated in Table 1." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 12</h3> <p class="intent_caption card-text Figure__text"><p>An illustrative example of the DET curve. The values of <em>FRR</em> and <em>FAR</em> of each point/threshold are calculated in <a href="#tbl1" class="text-link scroll_to Link intent_table_link Link__table" data-target="tbl1" alt="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value." title="Values of TP,FN,TN,FP,TPR,FPR,FNR, precision (PPV), and accuracy (Acc in %) of our ROC example when changes the threshold value.">Table 1</a>.</p></p> </div> </div> </div> </div> <div class="row pt-3"> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003013"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003013.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003013.tif&variant=thumbnail" alt="Results of our experiment. (a) ROC curve, (b) Precision-Recall curve." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003013" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003013.tif&variant=thumbnail" alt="Results of our experiment. (a) ROC curve, (b) Precision-Recall curve." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 13</h3> <p class="intent_caption card-text Figure__text"><p>Results of our experiment. (a) ROC curve, (b) Precision-Recall curve.</p></p> </div> </div> </div> <div class="col-sm-6 col-md-4"> <div class="intent_figure card mb-3 Figure" id="F_j.aci.2018.08.003014"> <a href="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003014.tif" target="emerald" class="intent_link d-block icon-enlarge p-2"> <img src="/insight/static/img/emerald-loading-wide-xl.gif" class="intent_image b-lazy card-img-top Figure__img" data-src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003014.tif&variant=thumbnail" alt="Confusion matrices of the three classes in our experiments." aria-description="The image opens in a new window" /> <noscript> <img id="F_j.aci.2018.08.003014" class="intent_image_noscript card-img-top Figure__img" src="/insight/proxy/img?link=resource/id/urn:emeraldgroup.com:asset:id:article:10_1016_j_aci_2018_08_003/urn:emeraldgroup.com:asset:id:binary:j.aci.2018.08.003014.tif&variant=thumbnail" alt="Confusion matrices of the three classes in our experiments." /> </noscript></a> <div class="card-block p-2 Figure__block"> <h3 class="h4 intent_label card-title Figure__title">Figure 14</h3> <p class="intent_caption card-text Figure__text"><p>Confusion matrices of the three classes in our experiments.</p></p> </div> </div> </div> </div> </section> <section class="intent_tables Tables Tables__article "> <div class="intent_table_block free dragscroll Table__block pb-2 mb-4 table-responsive table-wrap" id="tbl1"> <header class="Table__title"> <h2 class="intent_title"><label>Table 1</label></h2> </header> <caption class="intent_caption Table__caption"><p>Values of <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FN</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TN</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FP</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FPR</mml:mi><mml:mo>,</mml:mo><mml:mi mathvariant="italic">FNR</mml:mi></mml:mrow></mml:mrow></mml:math></span>, precision (<em>PPV</em>), and accuracy (<em>Acc</em> in %) of our ROC example when changes the threshold value.</p></caption> <table frame="hsides" class="table-data table" rules="groups"><colgroup><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/><col align="left" width="1*" span="1"/></colgroup><thead><tr><th align="left" rowspan="1">Threshold</th><th align="center" rowspan="1"><em>TP</em></th><th align="center" rowspan="1"><em>FN</em></th><th align="center" rowspan="1"><em>TN</em></th><th align="center" rowspan="1"><em>FP</em></th><th align="center" rowspan="1"><em>TPR</em></th><th align="center" rowspan="1"><em>FPR</em></th><th align="center" rowspan="1"><em>FNR</em></th><th align="center" rowspan="1"><em>PPV</em></th><th align="center" rowspan="1"><em>Acc</em></th></tr></thead><tbody><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>1</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mi>∞</mml:mi></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">–</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">50</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>2</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>82</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1.0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">55</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>3</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>80</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1.0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">60</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>4</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>75</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.67</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">55</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>5</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>70</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.75</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">60</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>6</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>62</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.80</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">65</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>7</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>60</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.83</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">70</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>8</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>54</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.71</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">65</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mn>9</mml:mn></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>50</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.63</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">60</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>10</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>49</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.67</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">65</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>11</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>45</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.60</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">60</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>12</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>40</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.64</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">65</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>13</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>39</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.58</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">60</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>14</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>37</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.5</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.62</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">65</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>15</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>32</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">4</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.6</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.57</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">60</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>16</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>30</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">3</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.7</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.53</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">55</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>17</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>26</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.50</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">50</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>18</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>23</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">2</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.8</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.53</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">55</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>19</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>21</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.50</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">50</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>20</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>19</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1.0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.9</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.53</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">55</td></tr><tr><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1"><span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:msub><mml:mi>t</mml:mi><mml:mrow><mml:mn>21</mml:mn></mml:mrow></mml:msub><mml:mo>=</mml:mo><mml:mtext>0</mml:mtext><mml:mtext>.</mml:mtext><mml:mn>10</mml:mn></mml:mrow></mml:mrow></mml:math></span></td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">10</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1.0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">1.0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">0.50</td><td class="Table__cell Table__body__cell" style="text-align: left;" rowspan="1">50</td></tr></tbody></table> </div> </section> <section class="intent_endnotes EndNotes mt-5 "> <h2 class="EndNote__title">Notes</h2> <div class="EndNote__section mt-2"> <div class="EndNote"><div class="Fn Fn__withLabel"><div class="Fn__label"><span class="label"><sup>1</sup></span></div><div class="Fn__nonLabelContent" id="fn001"><p>More details about these two metrics are in <a href="#sec002.2" class="text-link scroll_to Link intent_link" data-target="sec002.2" alt="" title="">Sections 2.2 and 2.5</a>.</p></div></div></div> <div class="EndNote"><div class="Fn Fn__withLabel"><div class="Fn__label"><span class="label"><sup>2</sup></span></div><div class="Fn__nonLabelContent" id="fn002"><p>More details about these two metrics are in <a href="#sec002.8" class="text-link scroll_to Link intent_link" data-target="sec002.8" alt="" title="">Section 2.8</a>.</p></div></div></div> <div class="EndNote"><div class="Fn Fn__withLabel"><div class="Fn__label"><span class="label"><sup>3</sup></span></div><div class="Fn__nonLabelContent" id="fn003"><p>As mentioned before <span class="inline-formula"><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow><mml:mrow><mml:mi mathvariant="italic">TPR</mml:mi><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi><mml:mo>+</mml:mo><mml:mi mathvariant="italic">FN</mml:mi></mml:mrow></mml:mfrac><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mi mathvariant="italic">TP</mml:mi></mml:mrow><mml:mi mathvariant="italic">P</mml:mi></mml:mfrac></mml:mrow></mml:mrow></mml:math></span> and both <em>TP</em> and <em>FN</em> are in the same column, and similarly <em>FNR</em>.</p></div></div></div> <div class="EndNote"><div class="Fn Fn__withLabel"><div class="Fn__label"><span class="label"><sup>4</sup></span></div><div class="Fn__nonLabelContent" id="fn004"><p>The AUC metric will be explained in <a href="#sec004" class="text-link scroll_to Link intent_link" data-target="sec004" alt="" title="">Section 4</a>.</p></div></div></div> <div class="EndNote"><div class="Fn Fn__withLabel"><div class="Fn__label"><span class="label"><sup>5</sup></span></div><div class="Fn__nonLabelContent" id="fn005"><p>A trapezoid is a 4-sided shape with two parallel sides.</p></div></div></div> <div class="EndNote"><div class="Fn Fn__withLabel"><div class="Fn__label"><span class="label"><sup>6</sup></span></div><div class="Fn__nonLabelContent" id="fn006"><p>More details about SVM can be found in [<a href="#ref024" class="text-link scroll_to Link intent_bibliographic_link Link__bibr" data-target="ref024" alt="" title="">24</a>].</p></div></div></div> </div> </section> <section class="References Chapter__references mt-5 "> <h2>References</h2> <div class="References__section"> <p class="Reference"> <span class="ref" id="ref001"><span class="Reference__label">[1]</span><span class="Reference__mixed-citation" data-publication-type="book"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">C.</span> <span class="Reference__surname">Blake</span></span></span>, <span class="Reference__source">Uci repository of machine learning databases</span>, <span class="Reference__year">1998</span>. <a href="http://www.ics.uci.edu/%7Emlearn/MLRepository.html" class="intent_external_link text-link ExtLink" rel="noopener noreferrer nofollow" target="_blank">http://www.ics.uci.edu/∼mlearn/MLRepository.html</a>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref002"><span class="Reference__label">[2]</span><span class="Reference__mixed-citation" data-publication-type="book"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">R.M.</span> <span class="Reference__surname">Bolle</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">J.H.</span> <span class="Reference__surname">Connell</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">S.</span> <span class="Reference__surname">Pankanti</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">N.K.</span> <span class="Reference__surname">Ratha</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">A.W.</span> <span class="Reference__surname">Senior</span></span></span>, <span class="Reference__source">Guide to biometrics</span>, <span class="Reference__publisher-name">Springer Science & Business Media</span>, <span class="Reference__year">2013</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref003"><span class="Reference__label">[3]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">S.</span> <span class="Reference__surname">Boughorbel</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">F.</span> <span class="Reference__surname">Jarray</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">M.</span> <span class="Reference__surname">El-Anbari</span></span></span>, <span class="Reference__article-title">Optimal classifier for imbalanced data using matthews correlation coefficient metric</span>, <span class="Reference__source">PLoS One</span> <span class="Reference__volume">12</span> (<span class="Reference__issue">6</span>) (<span class="Reference__year">2017</span>) <span class="Reference__elocation-id">e0177678</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref004"><span class="Reference__label">[4]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.P.</span> <span class="Reference__surname">Bradley</span></span></span>, <span class="Reference__article-title">The use of the area under the roc curve in the evaluation of machine learning algorithms</span>, <span class="Reference__source">Pattern Recogn</span>. <span class="Reference__volume">30</span> (<span class="Reference__issue">7</span>) (<span class="Reference__year">1997</span>) <span class="Reference__fpage">1145</span>–<span class="Reference__lpage">1159</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref005"><span class="Reference__label">[5]</span><span class="Reference__mixed-citation" data-publication-type="book-chapter"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">J.</span> <span class="Reference__surname">Davis</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">M.</span> <span class="Reference__surname">Goadrich</span></span></span>, <span class="Reference__chapter-title">The relationship between precision-recall and roc curves</span>, in: <span class="Reference__source">Proceedings of the 23rd International Conference on Machine Learning</span>, <span class="Reference__publisher-name">ACM</span>, <span class="Reference__year">2006</span>, pp. <span class="Reference__fpage">233</span>–<span class="Reference__lpage">240</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref006"><span class="Reference__label">[6]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">J.J.</span> <span class="Reference__surname">Deeks</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">D.G.</span> <span class="Reference__surname">Altman</span></span></span>, <span class="Reference__article-title">Diagnostic tests 4: likelihood ratios</span>, <span class="Reference__source">Brit. Med. J.</span> <span class="Reference__volume">329</span> (<span class="Reference__issue">7458</span>) (<span class="Reference__year">2004</span>) <span class="Reference__fpage">168</span>–<span class="Reference__lpage">169</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref007"><span class="Reference__label">[7]</span><span class="Reference__mixed-citation" data-publication-type="book"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">R.O.</span> <span class="Reference__surname">Duda</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">P.E.</span> <span class="Reference__surname">Hart</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">D.G.</span> <span class="Reference__surname">Stork</span></span>, <span class="contrib_block__etal">et al.</span></span>, <span class="Reference__source">Pattern Classification</span>, vol. <span class="Reference__volume">2</span>, <span class="Reference__publisher-name">Wiley</span>, <span class="Reference__publisher-loc">New York</span>, <span class="Reference__year">2001</span>. <span class="Reference__edition">second ed</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref008"><span class="Reference__label">[8]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">T.</span> <span class="Reference__surname">Fawcett</span></span></span>, <span class="Reference__article-title">An introduction to roc analysis</span>, <span class="Reference__source">Pattern Recogn. Lett.</span> <span class="Reference__volume">27</span> (<span class="Reference__issue">8</span>) (<span class="Reference__year">2006</span>) <span class="Reference__fpage">861</span>–<span class="Reference__lpage">874</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref009"><span class="Reference__label">[9]</span><span class="Reference__mixed-citation" data-publication-type="book-chapter"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">V.</span> <span class="Reference__surname">Garcia</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">R.A.</span> <span class="Reference__surname">Mollineda</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">J.S.</span> <span class="Reference__surname">Sanchez</span></span></span>, <span class="Reference__chapter-title">Theoretical analysis of a performance measure for imbalanced data</span>, in: <span class="Reference__source">20th International Conference on Pattern Recognition (ICPR)</span>, <span class="Reference__publisher-name">IEEE</span>, <span class="Reference__year">2010</span>, pp. <span class="Reference__fpage">617</span>–<span class="Reference__lpage">620</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref010"><span class="Reference__label">[10]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">D.J.</span> <span class="Reference__surname">Hand</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">R.J.</span> <span class="Reference__surname">Till</span></span></span>, <span class="Reference__article-title">A simple generalisation of the area under the roc curve for multiple class classification problems</span>, <span class="Reference__source">Mach. Learn.</span> <span class="Reference__volume">45</span> (<span class="Reference__issue">2</span>) (<span class="Reference__year">2001</span>) <span class="Reference__fpage">171</span>–<span class="Reference__lpage">186</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref011"><span class="Reference__label">[11]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">H.</span> <span class="Reference__surname">He</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">E.A.</span> <span class="Reference__surname">Garcia</span></span></span>, <span class="Reference__article-title">Learning from imbalanced data</span>, <span class="Reference__source">IEEE Trans. Knowledge Data Eng.</span> <span class="Reference__volume">21</span> (<span class="Reference__issue">9</span>) (<span class="Reference__year">2009</span>) <span class="Reference__fpage">1263</span>–<span class="Reference__lpage">1284</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref012"><span class="Reference__label">[12]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">V.</span> <span class="Reference__surname">López</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Fernández</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">S.</span> <span class="Reference__surname">Garća</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">V.</span> <span class="Reference__surname">Palade</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">F.</span> <span class="Reference__surname">Herrera</span></span></span>, <span class="Reference__article-title">An insight into classification with imbalanced data: empirical results and current trends on using data intrinsic characteristics</span>, <span class="Reference__source">Inf. Sci.</span> <span class="Reference__volume">250</span> (<span class="Reference__year">2013</span>) <span class="Reference__fpage">113</span>–<span class="Reference__lpage">141</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref013"><span class="Reference__label">[13]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Maratea</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Petrosino</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">M.</span> <span class="Reference__surname">Manzo</span></span></span>, <span class="Reference__article-title">Adjusted f-measure and kernel scaling for imbalanced data learning</span>, <span class="Reference__source">Inf. Sci.</span> <span class="Reference__volume">257</span> (<span class="Reference__year">2014</span>) <span class="Reference__fpage">331</span>–<span class="Reference__lpage">341</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref014"><span class="Reference__label">[14]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">B.W.</span> <span class="Reference__surname">Matthews</span></span></span>, <span class="Reference__article-title">Comparison of the predicted and observed secondary structure of t4 phage lysozyme</span>, <span class="Reference__source">Biochim. Biophys. Acta</span> <span class="Reference__volume">405</span> (<span class="Reference__issue">2</span>) (<span class="Reference__year">1975</span>) <span class="Reference__fpage">442</span>–<span class="Reference__lpage">451</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref015"><span class="Reference__label">[15]</span><span class="Reference__mixed-citation" data-publication-type="book-chapter"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">C.E.</span> <span class="Reference__surname">Metz</span></span></span>, <span class="Reference__chapter-title">Basic principles of roc analysis</span>, in: <span class="Reference__source">Seminars in nuclear medicine</span>, vol. <span class="Reference__volume">8</span>, <span class="Reference__publisher-name">Elsevier</span>, <span class="Reference__year">1978</span>, pp. <span class="Reference__fpage">283</span>–<span class="Reference__lpage">298</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref016"><span class="Reference__label">[16]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">D.M.</span> <span class="Reference__surname">Powers</span></span></span>, <span class="Reference__article-title">Evaluation: from precision, recall and f-measure to roc</span>, <span class="Reference__source">informedness, markedness and correlation</span> <span class="Reference__volume">2</span> (<span class="Reference__issue">1</span>) (<span class="Reference__year">2011</span>) <span class="Reference__fpage">37</span>–<span class="Reference__lpage">63</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref017"><span class="Reference__label">[17]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">T.</span> <span class="Reference__surname">Saito</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">M.</span> <span class="Reference__surname">Rehmsmeier</span></span></span>, <span class="Reference__article-title">The precision-recall plot is more informative than the roc plot when evaluating binary classifiers on imbalanced datasets</span>, <span class="Reference__source">PLoS One</span> <span class="Reference__volume">10</span> (<span class="Reference__issue">3</span>) (<span class="Reference__year">2015</span>) <span class="Reference__elocation-id">e0118432</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref018"><span class="Reference__label">[18]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Shaffi</span></span></span>, <span class="Reference__article-title">Measures derived from a 2 x 2 table for an accuracy of a diagnostic test</span>, <span class="Reference__source">J. Biometr. Biostat.</span> <span class="Reference__volume">2</span> (<span class="Reference__year">2011</span>) <span class="Reference__fpage">1</span>–<span class="Reference__lpage">4</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref019"><span class="Reference__label">[19]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">S.</span> <span class="Reference__surname">Shaikh</span></span></span>, <span class="Reference__article-title">Measures derived from a 2 x 2 table for an accuracy of a diagnostic test</span>, <span class="Reference__source">J. Biometr. Biostat.</span> <span class="Reference__volume">2</span> (<span class="Reference__year">2011</span>) <span class="Reference__fpage">128</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref020"><span class="Reference__label">[20]</span><span class="Reference__mixed-citation" data-publication-type="book-chapter"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">M.</span> <span class="Reference__surname">Sokolova</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">N.</span> <span class="Reference__surname">Japkowicz</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">S.</span> <span class="Reference__surname">Szpakowicz</span></span></span>, <span class="Reference__chapter-title">Beyond accuracy, f-score and roc: a family of discriminant measures for performance evaluation</span>, in: <span class="Reference__source">Australasian Joint Conference on Artificial Intelligence</span>, <span class="Reference__publisher-name">Springer</span>, <span class="Reference__year">2006</span>, pp. <span class="Reference__fpage">1015</span>–<span class="Reference__lpage">1021</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref021"><span class="Reference__label">[21]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">M.</span> <span class="Reference__surname">Sokolova</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">G.</span> <span class="Reference__surname">Lapalme</span></span></span>, <span class="Reference__article-title">A systematic analysis of performance measures for classification tasks</span>, <span class="Reference__source">Inf. Process. Manage.</span> <span class="Reference__volume">45</span> (<span class="Reference__issue">4</span>) (<span class="Reference__year">2009</span>) <span class="Reference__fpage">427</span>–<span class="Reference__lpage">437</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref022"><span class="Reference__label">[22]</span><span class="Reference__mixed-citation" data-publication-type="book"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Srinivasan</span></span></span>, <span class="Reference__source">Note on the location of optimal classifiers in n-dimensional roc space</span>. <span class="Reference__comment">Technical Report PRG-TR-2-99, Oxford University Computing Laboratory, Oxford, England</span>, <span class="Reference__year">1999</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref023"><span class="Reference__label">[23]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Tharwat</span></span></span>, <span class="Reference__article-title">Principal component analysis-a tutorial</span>, <span class="Reference__source">Int. J. Appl. Pattern Recogn.</span> <span class="Reference__volume">3</span> (<span class="Reference__issue">3</span>) (<span class="Reference__year">2016</span>) <span class="Reference__fpage">197</span>–<span class="Reference__lpage">240</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref024"><span class="Reference__label">[24]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Tharwat</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">A.E.</span> <span class="Reference__surname">Hassanien</span></span></span>, <span class="Reference__article-title">Chaotic antlion algorithm for parameter optimization of support vector machine</span>, <span class="Reference__source">Appl. Intelligence</span> <span class="Reference__volume">48</span> (<span class="Reference__issue">3</span>) (<span class="Reference__year">2018</span>) <span class="Reference__fpage">670</span>–<span class="Reference__lpage">686</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref025"><span class="Reference__label">[25]</span><span class="Reference__mixed-citation" data-publication-type="journal"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">A.</span> <span class="Reference__surname">Tharwat</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">Y.S.</span> <span class="Reference__surname">Moemen</span></span>, <span class="Reference__string-name"><span class="Reference__given-names">A.E.</span> <span class="Reference__surname">Hassanien</span></span></span>, <span class="Reference__article-title">Classification of toxicity effects of biotransformed hepatic drugs using whale optimized support vector machines</span>, <span class="Reference__source">J. Biomed. Inf.</span> <span class="Reference__volume">68</span> (<span class="Reference__year">2017</span>) <span class="Reference__fpage">132</span>–<span class="Reference__lpage">149</span>.</span></span> </p> <p class="Reference"> <span class="ref" id="ref026"><span class="Reference__label">[26]</span><span class="Reference__mixed-citation" data-publication-type="web"><span class="Reference__person-group" data-person-group-type="author"><span class="Reference__string-name"><span class="Reference__given-names">K.H.</span> <span class="Reference__surname">Zou</span></span></span>, <span class="Reference__article-title">Receiver operating characteristic (roc) literature research</span>, <span class="Reference__year">2002</span>. <span class="Reference__comment">On-line bibliography available from:</span> <a href="http://splweb.bwh.harvard.edu8000" class="intent_external_link text-link ExtLink" rel="noopener noreferrer nofollow" target="_blank">http://splweb.bwh.harvard.edu8000</a>.</span></span> </p> </div> </section> <section class="Acknowledgements mt-5"> <h2 class="intent_acknowledgements">Acknowledgements</h2> <span class="ack" id="ack"><p>Publishers note: The publisher wishes to inform readers that the article “Classification assessment methods” was originally published by the previous publisher of Applied Computing and Informatics and the pagination of this article has been subsequently changed. There has been no change to the content of the article. This change was necessary for the journal to transition from the previous publisher to the new one. The publisher sincerely apologises for any inconvenience caused. To access and cite this article, please use Tharwat, A. (2020), “Classification assessment methods”, New England Journal of Entrepreneurship. Vol. 17 No. 1, pp. 168-192. The original publication date for this paper was 21/08/2018.</p></span> </section> <section class="AuthorNotes "> <h2>Corresponding author</h2> <div class="author-notes"><span class="corresp" id="cor1">Alaa Tharwat can be contacted at: <a href="mailto:aothman@fb2.fra-uas.de" class="intent_external_link text-link ExtLink" rel="noopener noreferrer nofollow" target="_blank">aothman@fb2.fra-uas.de</a></span></div> </section> </div> </div> <div class="col-12 col-md-2"></div> </div> <div class="col-12 col-md-3"> <h2 class="trendmd-replacement-heading mt-4">Related articles</h2> <div id="trendmd-suggestions"></div> </div> </div> </div> </main> <flash message=""></flash> <div id="feedback-strip" class="header-feedback dropdown "> <div class="header-feedback__link "> <button type="button" class="intent_header_feedback_link header-feedback__link-button collapsed" data-toggle="collapse" data-target="#header-feedback-content" aria-expanded="false" > <div class="header-feedback__link-left font-weight-bold text-left"> <span id="header-feedback__link-text">Support & Feedback</span> <span id="header-feedback__link-caret" class="toggleCaret fas fa-caret-up fa-lg ml-1"></span> </div> </button> <a href="#" class="header-feedback__link-cookie-button text-white intent_cookie_manage" data-toggle="modal" data-target="#manage-cookies-modal"> Manage cookies </a> </div> <div id="header-feedback-content" class="intent_header_feedback_content header-feedback__content collapse pt-4 pb-4 text-center text-white"> <div class="container"> <div class="row px-lg-5"> <div class="col-12 col-md-6 text-left mb-4 mb-md-0"> <h3>All feedback is valuable</h3> <p>Please <a href="https://eu.surveymonkey.com/r/DGMS7Z8" class="text-light" target="_blank">share your general feedback</a></p> </div> <div class="col-12 col-md-6 text-left"> <h3>Report an issue or find answers to frequently asked questions</h3> <p>Contact <a href="https://emeraldpublishinggroup.freshdesk.com/support/home" class="text-light" target="_blank">Customer Support</a></p> </div> </div> </div> </div> </div> <div id="feedback-underlay" class="feedback-underlay"> </div> <footer role="contentinfo" class="bg-dark pt-4 text-center text-white"> <div class="container"> <div class="row px-lg-5 pb-3"> <div class="col-12 col-md-3 text-md-left pl-md-2 pr-md-4 pb-4 pb-md-0 text-white"> <img data-src="/insight/static/img/emerald_publishing_logo-white.svg" class="img-fluid w-75 mw-sm-100 mw-xl-50 b-lazy" alt="Emerald logo" /> <div class="mt-3 ml-md-n1 d-flex flex-row justify-content-center justify-content-md-start align-self-center"> <ul class="list-inline my-2 mt-md-2 mt-lg-0"> <li class="list-inline-item"><a href="https://twitter.com/EmeraldGlobal" target="_blank" rel="noopener noreferrer" title="Twitter (opens in new window)" aria-label="Twitter (opens in new window)" class="text-white font-size-small p-1"><span role="img" class="fab fa-twitter"></span></a></li> <li class="list-inline-item"><a href="https://www.facebook.com/EmeraldPublishingImpact/" target="_blank" rel="noopener noreferrer" title="Facebook (opens in new window)" aria-label="Facebook (opens in new window)" class="text-white font-size-small p-1"><span role="img" class="fab fa-facebook-f"></span></a></li> <li class="list-inline-item"><a href="https://www.linkedin.com/company/emerald-group-publishing-limited" target="_blank" rel="noopener noreferrer" title="LinkedIn (opens in new window)" aria-label="LinkedIn (opens in new window)" class="text-white font-size-small p-1"><span role="img" class="fab fa-linkedin-in"></span></a></li> <li class="list-inline-item"><a href="https://www.youtube.com/user/EmeraldPublishing67" target="_blank" rel="noopener noreferrer" title="YouTube (opens in new window)" aria-label="YouTube (opens in new window)" class="text-white font-size-small p-1"><span role="img" class="fab fa-youtube-square"></span></a></li> </ul> </div> <div class="text-white font-size-xsmall font-weight-light">© 2024 Emerald Publishing Limited. All rights reserved, including rights for text and data mining, artificial intelligence training and similar technologies.</div> </div>  <div class="col-12 col-md-3 text-md-left pl-md-5 pb-4 pb-md-0"> <h2 class="h4 mb-md-3">Services</h2> <ul class="list-unstyled my-0"> <li><a href="https://www.emeraldgrouppublishing.com/services/authors" class="mixpanel_link intent_footer_link_services_authors text-white font-size-small font-weight-light" target="_blank" rel="noopener noreferrer" aria-label="Authors (opens in new window)" title="Authors (opens in new window)">Authors</a></li> <li><a href="https://www.emeraldgrouppublishing.com/services/journal-editors" class="mixpanel_link intent_footer_link_services_editors text-white font-size-small font-weight-light" target="_blank" rel="noopener noreferrer" aria-label="Editors (opens in new window)" title="Editors (opens in new window)">Editors</a></li> <li><a href="https://www.emeraldgrouppublishing.com/services/librarians" class="mixpanel_link intent_footer_link_services_librarians text-white font-size-small font-weight-light" target="_blank" rel="noopener noreferrer" aria-label="Librarians (opens in new window)" title="Librarians (opens in new window)">Librarians</a></li> <li><a href="https://www.emeraldgrouppublishing.com/services/researchers" class="mixpanel_link intent_footer_link_services_research text-white font-size-small font-weight-light" target="_blank" rel="noopener noreferrer" aria-label="Researchers (opens in new window)" title="Researchers (opens in new window)">Researchers</a></li> <li><a href="https://www.emeraldgrouppublishing.com/services/reviewers" class="mixpanel_link intent_footer_link_services_reviewers text-white font-size-small font-weight-light" target="_blank" rel="noopener noreferrer" aria-label="Reviewers (opens in new window)" title="Reviewers (opens in new window)">Reviewers</a></li> </ul> </div> <div class="col-12 col-md-3 text-md-left pb-4 pb-md-0"> <h2 class="h4 mb-md-3">About</h2> <ul class="list-unstyled my-0"> <li><a href="https://www.emeraldgrouppublishing.com/about/" target="_blank" rel="noopener noreferrer" aria-label="About Emerald (opens in new window)" title="About Emerald (opens in new window)" class="mixpanel_link intent_footer_link_about_emerald text-white font-size-small font-weight-light">About Emerald</a></li> <li><a href="https://careers.emeraldpublishing.com" target="_blank" rel="noopener noreferrer" aria-label="Working for Emerald (opens in new window)" title="Working for Emerald (opens in new window)" class="intent_footer_link_about_working text-white font-size-small font-weight-light">Working for Emerald</a></li> <li><a href="https://www.emeraldgrouppublishing.com/about/contact-us" target="_blank" rel="noopener noreferrer" aria-label="Contact us (opens in new window)" title="Contact us (opens in new window)" class="mixpanel_link intent_footer_link_about_contact text-white font-size-small font-weight-light">Contact us</a></li> <li><a href="/insight/sitemap/publications" class="intent_footer_link_publication_sitemap text-white font-size-small font-weight-light">Publication sitemap</a></li> </ul> </div> <div class="col-12 col-md-3 text-md-left"> <h2 class="h4 mb-md-3">Policies and information</h2> <ul class="list-unstyled my-0"> <li><a href="/insight/site-policies" class="intent_footer_link_policies_privacy text-white font-size-small font-weight-light">Privacy notice</a></li> <li><a href="/insight/site-policies" class="intent_footer_link_policies_site_policies text-white font-size-small font-weight-light">Site policies</a></li> <li><a href="https://www.emeraldgrouppublishing.com/about/policies-and-information/modern-slavery-act" target="_blank" rel="noopener noreferrer" aria-label="Modern Slavery Act (opens in new window)" title="Modern Slavery Act (opens in new window)" class="mixpanel_link intent_footer_link_policies_modern_slavery text-white font-size-small font-weight-light">Modern Slavery Act</a></li> <li><a href="https://www.emeraldgrouppublishing.com/sites/default/files/2020-08/MCB-Pension-Chair-Statement-2019.pdf" target="_blank" rel="noopener noreferrer" aria-label="Chair of Trustees governance statement (opens in new window, PDF, 511 KB)" title="Chair of Trustees governance statement (opens in new window, PDF, 511 KB)" class="mixpanel_link intent_footer_link_trustees_governance_statement text-white font-size-small font-weight-light">Chair of Trustees governance statement</a></li> <li><a href="/insight/accessibility" class="intent_footer_link_accessibility text-white font-size-small font-weight-light">Accessibility</a></li> </ul> </div> </div> </div> </footer> <div id="topscroll"> <top-scroll text="Back to top" visibleoffset="500"></top-scroll> </div> </div> <script type="text/javascript" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> window.subDir = "/insight/" function document_ready (callback) { if (document.readyState !== 'loading') { callback() } else { document.addEventListener('DOMContentLoaded', callback) } } // Replacement for $(element).on(event, selector, handler), as addEventListener // does not support the selector parameter. function add_event_listener(element, event, selector, handler) { if (element === null) { return } element.addEventListener(event, function (e) { for (let target = e.target; target && target !== this; target = target.parentNode) { if (target instanceof Element && target.matches(selector)) { handler.call(target, e) } } }, false) } </script> <script src="/insight/static/js/manifest.js?id=bf526d8f397db3044526d8a29f4ab821" defer></script> <script src="/insight/static/js/vendor.js?id=1ad3c16f98039b6cb7c17989d3b6766c" defer></script> <script src="/insight/static/js/app.js?id=226e64afdb01e284f81639e3ea784e4d" defer></script> <script src="/insight/static/js/vendor/dragscroll.js?id=150f32f78896241390129daaff2ebed9" defer></script> <script src="/insight/static/js/mixpanel/mixpanel.js?id=a8237b34f4932b653178db5ff7c26496" defer></script> <script type="text/javascript" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> document_ready(function () { // enable bootstrap tooltips globally $('[data-toggle="tooltip"]').tooltip() }) </script> <script type="text/javascript" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> document_ready(function () { document.querySelector('.header-feedback__link-button').addEventListener('click', function () { $('#header-feedback__link-caret').toggleClass('fa-caret-up fa-caret-down'); if( $('#header-feedback__link-text').text() == 'Close') { $('#header-feedback__link-text').text('Support & Feedback'); $('#feedback-underlay').hide(); } else { $('#header-feedback__link-text').text('Close'); $('#feedback-underlay').show(); } }); let scrollMagnet = $('.scroll-magnet-item') if (typeof scrollMagnet[0] === 'undefined') { return } $.get(window.subDir + 'api/banner', function (data) { if (data.status === 'Down') { let bodyElement = $('body') bodyElement.attr('data-offset', 215) let scrollMagnet = $('.scroll-magnet-item') if (typeof scrollMagnet[0] === 'undefined') { return } let observer = new MutationObserver(function () { const scrollingBanner = 'is-scrolling--support-banner' const isScrolling = 'is-scrolling' if (scrollMagnet.hasClass(isScrolling) && !scrollMagnet.hasClass(scrollingBanner)) { scrollMagnet.addClass(scrollingBanner) } if (!scrollMagnet.hasClass(isScrolling) && scrollMagnet.hasClass(scrollingBanner)) { scrollMagnet.removeClass(scrollingBanner) } }) observer.observe(scrollMagnet[0], { attributes: true, attributeFilter: ['class'], childList: false, characterData: false }) } }) }) </script> <script src="/insight/static/js/mixpanel/mixpanel-cookies.js?id=e74fd8be58799c9220a39e93a7051c7f" defer></script> <div id="shivPlaceholder" tabindex="-1" class="shivContents position-relative text-center mt-3 mb-3 px-5 py-3 pt-4 border table-responsive" style="display: none"> <div style="z-index: 999; right: 1rem; top: 0;" class="position-absolute shiv-header row text-right"> <button style="font-size: 2.1rem; width:2.1rem; height:2.1rem; overflow: hidden;" type="button" class="close" title="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="free dragscroll"> <div class="shivContent d-inline-block m-auto text-left position-relative"></div> </div> </div> <script src="/insight/static/js/emerald/js_shiv.js?id=82575e0833ba874edd4978c3da18b553" defer></script> <script src="/insight/static/js/emerald/js_toc.js?id=fecef861f17178bd72eca612ca38366a" defer></script> <script type="text/javascript" id="trendmd-block" nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> if (window.cookieHelper.areFunctionalCookiesEnabled()) { /** * If we just included the script, it would execute at a deferred point in time, * regardless of our cookie setting, therefore it's added as a script element only * on condition that functional cookies have actually been enabled **/ const trendmdBlock = document.getElementById("trendmd-block"); const trendmdScriptTag = document.createElement("script"); trendmdScriptTag.src ='//js.trendmd.com/trendmd.min.js'; trendmdScriptTag.setAttribute('data-trendmdconfig', '{"element":"#trendmd-suggestions"}'); trendmdScriptTag.setAttribute('defer', 'true'); trendmdBlock.append(trendmdScriptTag); const eventName = 'TrendMD' function mixpanelProperties (el) { const url = (el.href || '').split('?')[0] return { 'Article name': el.text, 'Format': url.endsWith('.pdf') ? 'PDF' : 'HTML', 'Link Type': url.includes('emerald') ? 'Internal' : 'External', url: el.href } } let alreadySeenLinks = [] isReady(() => { const suggestions = document.querySelector('#trendmd-suggestions') suggestions.classList.add('d-none') const suggestionsHeader = document.querySelector('.trendmd-replacement-heading') suggestionsHeader.classList.add('d-none') add_event_listener(suggestions, 'DOMNodeInserted', '.trendmd-widget-list', event => { const links = Array.from(event.currentTarget.querySelectorAll('a.trendmd-widget-list-item__link')) const newLinks = links.filter(l => !alreadySeenLinks.includes(l)) // add new links to the already seen list so we can make sure we don't attach listeners to them more than once alreadySeenLinks = alreadySeenLinks.concat(newLinks) newLinks.forEach(link => mixpanelTrackLinkClick(link, eventName, mixpanelProperties)) suggestionsHeader.classList.remove('d-none') suggestions.classList.remove('d-none') }) }); } </script> <script type='text/javascript' src='https://d1bxh8uas1mnw7.cloudfront.net/assets/embed.js' defer></script> <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> function trackingEnabled() { const isLockss = (navigator.userAgent === 'LOCKSS cache'); const shouldSkipTracking = (window.location.href.indexOf('skipTracking=true') > -1); return !isLockss && !shouldSkipTracking; }; </script> <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> const mixPanelJson = {"URI":"\/content\/doi\/10.1016\/j.aci.2018.08.003\/full\/html","Format":"HTML","Content ID \/ DOI":"10.1016\/j.aci.2018.08.003","Content Title":"Classification assessment methods","Parent Title":"Applied Computing and Informatics","Year Of Publication":"2020","Publication Date":"30 July 2020","Access Method":"User","Access Type":"OA_gold","Parent Type":"journal","Print ISSN":"2634-1964","Online ISSN":"2210-8327","Parent ID \/ DOI":"10.1108\/ACI","Content Type":"article"}; const mixPanelCurrentPageEvent = 'Download'; let teachingNotesAlreadyClicked = false; let caseStudyAlreadyClicked = false; let articleAlreadyClicked = false; function mixpanelTrack(eventName, mixPanelData) { storeOrPushMixpanelEvent('track', eventName, mixPanelData); mixpanelUpdateUserDataForEvent(eventName); } function trackTeachingNoteDownload() { if (teachingNotesAlreadyClicked) { return; } teachingNotesAlreadyClicked = true; const mixPanelData = JSON.parse(JSON.stringify(mixPanelJson)); mixPanelData['Content Type'] = 'Teaching Note'; mixpanelTrack(mixPanelCurrentPageEvent, mixPanelData); } function trackCaseStudyDownload() { if (caseStudyAlreadyClicked) { return; } caseStudyAlreadyClicked = true; const mixPanelData = JSON.parse(JSON.stringify(mixPanelJson)); mixPanelData['Content Type'] = 'case study'; mixpanelTrack(mixPanelCurrentPageEvent, mixPanelData); } function trackArticleDownload() { if (articleAlreadyClicked) { return; } articleAlreadyClicked = true; const mixPanelData = JSON.parse(JSON.stringify(mixPanelJson)); mixPanelData['Content Type'] = 'article'; mixpanelTrack(mixPanelCurrentPageEvent, mixPanelData); } function eventInterceptInit() { add_event_listener(window, 'click', 'a.clickIntercept', function () { const $this = $(this); switch($this.data("contentType")) { case 'Case Study': trackCaseStudyDownload(); break; case 'Teaching Note': trackTeachingNoteDownload(); break; case 'Article': trackArticleDownload(); break; default: const mixPanelData = JSON.parse(JSON.stringify(mixPanelJson)); const clickEventName = $this.data("eventName"); mixpanelTrack(clickEventName, mixPanelData); } }) } // Check if the tabs were already clicked on page land, if it was then track the relevant event function checkSelectedTabs() { const teachingNotes = document.querySelector('#teaching-notes') if (teachingNotes != null && teachingNotes.getAttribute('aria-selected') === 'true') { trackTeachingNoteDownload() return } const cases = document.querySelector('#cases') if (cases != null && cases.getAttribute('aria-selected') === 'true') { trackCaseStudyDownload() return } const articles = document.querySelector('#articles') if (articles != null && articles.getAttribute('aria-selected') === 'true') { trackArticleDownload() return } mixpanelTrack(mixPanelCurrentPageEvent, mixPanelJson) } (function(){ if (trackingEnabled()) { checkSelectedTabs(); eventInterceptInit(); }; })(); </script> <script nonce="vw/M1EJtfJm4SIes+W5PaCZGFTwAGtQUq/fcbk8QS9ZWpPve9CW6MVbxhTZeopQWzWf9HgCW2H9DE5IQs2sQjQ=="> var _ll = _ll || []; function pushToLL(event, data) { _ll.push([event, data]); } function buildUrl(path, params = {}) { const root = "/insight/"; const queryString = Object.keys(params) .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(params[key])}`) .join('&'); return `${root}${path}?${queryString}`; } function counterTrack(endpoint) { pushToLL('setEndpoint', endpoint); pushToLL('setDefaults', {"pid":"0959a6c7150c5e64","plname":"Emerald Publishing Limited"}); let counterJson = {"tm":{"title":"Applied Computing and Informatics","dt":"journal","issnp":"2634-1964","issno":"2210-8327","doi":"10.1108\/ACI"},"im":{"doi":"10.1016\/j.aci.2018.08.003","title":"Classification assessment methods","yop":"2020","pdate":"30 July 2020","dt":"article","stype":"article"},"aid":"00001","aname":"Guest User","sid":"JDsytJmKE5aVUjttOxdpqfwYEihQ2uHr6KyH74fu","uri":"\/content\/doi\/10.1016\/j.aci.2018.08.003\/full\/html","fmt":"HTML","referrer":"","am":"regular","at":"OA_gold"}; return new Promise((resolve, reject) => { pushToLL('trackItemRequest', counterJson); resolve(); // Resolve immediately if no API call is made }); } function counterLoadScript() { const d = document; const g = d.createElement('script'); const s = d.getElementsByTagName('script')[0]; const libLynxPrefixOne = "https://connect.lib"; const libLynxPrefixTwo = "lynx.com"; const libLynxSuffix = "/log/js/counter5.min.js"; const libLynxSrc = `${libLynxPrefixOne}${libLynxPrefixTwo}${libLynxSuffix}`; g.type = 'text/javascript'; g.async = true; g.id = 'counter_script'; g.defer = true; g.src = libLynxSrc; s.parentNode.insertBefore(g, s); } function counterGetEndpoint() { const baseUrlOne = "https://connect.lib".replace('https:','').replace('http:',''); const baseUrlTwo = "lynx.com"; return baseUrlOne + baseUrlTwo + "/log/counter"; } document_ready(async () => { if (trackingEnabled()) { await counterTrack(counterGetEndpoint()); counterLoadScript(); } }); </script> </body> </html>

CINXE.COM

Classification assessment methods | Emerald Insight