CINXE.COM

<!doctype html><html lang="en" class="web tlp-clear" data-studio-config="eyJ4aHJDcmVkZW50aWFscyI6ZmFsc2UsInhockhlYWRlcnMiOnt9fQo="><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /><title>The EPSS Model</title> <meta property="og:title" content="The EPSS Model" /> <meta property="og:type" content="website" /> <meta property="og:image" content="https://www.first.org/_/img/first-big-icon.png" /> <meta property="og:url" content="https://www.first.org/epss/model" /> <meta property="og:site_name" content="FIRST — Forum of Incident Response and Security Teams" /> <meta property="fb:profile_id" content="296983660669109" /> <meta property="twitter:card" content="summary" /> <meta property="twitter:site" content="@FIRSTdotOrg" /><meta name="viewport" content="initial-scale=1,maximum-scale=1.0,user-scalable=no" /><link rel="icon" type="image/png" href="/1st.png" /><link rel="apple-touch-icon" sizes="128x128" href="/favicon.png" /><link rel="stylesheet" type="text/css" href="/_/web.css?20241031194005" /></head><body><header><div id="header" data-studio="CU52CV1W8g"><div id="c3" data-studio="Yu8FjCC11g"><div id="topbar"> <div class="sites right"> <ul> <li><a href="https://support.first.org" class="kb-datalist"><img src="/_/img/icon-portal_support.svg" alt="FIRST Support" title="FIRST Support" /></a></li> <li><a href="https://portal.first.org" class="button"><span class="no-tiny">Member </span>Portal</a></li> </ul> </div> <div class="first-logo"> <p><a href="/"><img src="/_/img/first-org-simple-negative.svg" alt="FIRST.Org" title="FIRST" /></a></p> </div> <div class="nav"> <ul class="navbar"><li><a href="/about">About FIRST</a><ul><li><a href="/about/mission">Mission Statement</a></li><li><a href="/about/history">History</a></li><li><a href="/about/sdg">Sustainable Development Goals</a></li><li><a href="/about/organization">Organization</a><ul><li><a href="/about/organization/directors">Board of Directors</a></li><li><a>Operations Team</a><ul><li><a href="/about/organization/ccb">Community & Capacity Building</a></li><li><a href="/about/organization/events">Event Office</a></li><li><a href="/about/organization/executive-director">Executive Director</a></li><li><a href="/about/organization/infrastructure">Infrastructure</a></li><li><a href="/about/organization/secretariat">Secretariat</a></li></ul></li><li><a href="/about/organization/committees">Committees</a><ul><li><a href="/about/organization/committees/compensation-committee">Compensation Committee</a></li><li><a href="/about/organization/committees/conference-program-committee">Conference Program Committee</a></li><li><a href="/about/organization/committees/membership-committee">Membership Committee</a></li><li><a href="/about/organization/committees/rules-committee">Rules Committee</a></li><li><a href="/about/organization/committees/standards">Standards Committee</a></li></ul></li><li><a href="/events/agm">Annual General Meeting</a></li><li><a href="/about/organization/reports">Annual Reports and Tax Filings</a></li></ul></li><li><a href="/about/policies">FIRST Policies</a><ul><li><a href="/about/policies/anti-corruption">Anti-Corruption Policy</a></li><li><a href="/about/policies/antitrust">Antitrust Policy</a></li><li><a href="/about/policies/bylaws">Bylaws</a></li><li><a href="/about/policies/board-duties">Board duties</a></li><li><a href="/about/bugs">Bug Bounty Program</a></li><li><a href="/about/policies/code-of-conduct">Code of Conduct</a></li><li><a href="/about/policies/conflict-policy">Conflict of Interest Policy</a></li><li><a href="/about/policies/doc-rec-retention-policy">Document Record Retention and Destruction Policy</a></li><li><a href="/newsroom/policy">FIRST Press Policy</a></li><li><a href="/about/policies/gen-event-reg-refund-policy">General Event Registration Refund Policy</a></li><li><a href="/about/policies/event-site-selection">Guidelines for Site Selection for all FIRST events</a></li><li><a href="/identity">Identity & Logo Usage</a></li><li><a href="/about/policies/mailing-list">Mailing List Policy</a></li><li><a href="/about/policies/media">Media Policy</a></li><li><a href="/about/policies/privacy">Privacy Policy</a></li><li><a href="/about/policies/registration-terms-conditions">Registration Terms & Conditions</a></li><li><a href="/about/policies/terms">Services Terms of Use</a></li><li><a href="/about/policies/standards">Standards Policy</a></li><li><a href="/about/policies/diversity">Statement on Diversity & Inclusion</a></li><li><a href="/about/policies/translation-policy">Translation Policy</a></li><li><a href="/about/policies/travel-policy">Travel Policy</a></li><li><a href="/about/policies/uniform-ipr">Uniform IPR Policy</a></li><li><a href="/about/policies/whistleblower-policy">Whistleblower Protection Policy</a></li></ul></li><li><a href="/about/partners">Partnerships</a><ul><li><a href="/global/partners">Partners</a></li><li><a href="/global/friends">Friends of FIRST</a></li><li><a href="/global/supporters/">FIRST Supporters</a></li><li><a href="/about/sponsors">Sponsors</a></li></ul></li><li><a href="/newsroom">Newsroom</a><ul><li><a href="/newsroom/news">What's New</a></li><li><a href="/newsroom/releases">Press Releases</a></li><li><a href="/newsroom/news/media">In the News</a></li><li><a href="/podcasts">Podcasts</a><ul><li><a href="/newsroom/news/first-impressions/">FIRST Impressions Podcast</a></li><li><a href="/newsroom/news/podcasts/">FIRSTCON Podcast</a></li></ul></li><li><a href="/newsroom/newsletters">Newsletters</a></li><li><a href="/newsroom/policy">FIRST Press Policy</a></li></ul></li><li><a href="/about/procurement">Procurement</a></li><li><a href="/about/jobs/">Jobs</a></li><li><a href="/contact">Contact</a></li></ul></li><li><a href="/members">Membership</a><ul><li><a href="/membership/">Becoming a Member</a><ul><li><a href="/membership/process">Membership Process for Teams</a></li><li><a href="/membership/process-liaisons">Membership Process for Liaisons</a></li><li><a href="/membership/#Fees">Membership Fees</a></li></ul></li><li><a href="/members/teams">FIRST Teams</a></li><li><a href="/members/liaisons">FIRST Liaisons</a></li><li><a href="/members/map">Members around the world</a></li></ul></li><li><a href="/global">Initiatives</a><ul><li><a href="/global/sigs">Special Interest Groups (SIGs)</a><ul><li><a href="/global/sigs/framework">SIGs Framework</a></li><li><a href="/global/sigs/academicsec" class="borderb">Academic Security SIG</a></li><li><a href="/global/sigs/ai-security">AI Security SIG</a></li><li><a href="/global/sigs/automation">Automation SIG</a></li><li><a href="/global/sigs/bigdata">Big Data SIG</a></li><li><a href="/cvss">Common Vulnerability Scoring System (CVSS-SIG)</a><ul><li><a href="/cvss/calculator/4.0">Calculator</a></li><li><a href="/cvss/v4.0/specification-document">Specification Document</a></li><li><a href="/cvss/v4.0/user-guide">User Guide</a></li><li><a href="/cvss/v4.0/examples">Examples</a></li><li><a href="/cvss/v4.0/faq">Frequently Asked Questions</a></li><li><a href="/cvss/v4-0">CVSS v4.0 Documentation & Resources</a><ul><li><a href="/cvss/calculator/4.0">CVSS v4.0 Calculator</a></li><li><a href="/cvss/v4.0/specification-document">CVSS v4.0 Specification Document</a></li><li><a href="/cvss/v4.0/user-guide">CVSS v4.0 User Guide</a></li><li><a href="/cvss/v4.0/examples">CVSS v4.0 Examples</a></li><li><a href="/cvss/v4.0/faq">CVSS v4.0 FAQ</a></li></ul></li><li><a href="/cvss/v3-1">CVSS v3.1 Archive</a><ul><li><a href="/cvss/calculator/3.1">CVSS v3.1 Calculator</a></li><li><a href="/cvss/v3.1/specification-document">CVSS v3.1 Specification Document</a></li><li><a href="/cvss/v3.1/user-guide">CVSS v3.1 User Guide</a></li><li><a href="/cvss/v3.1/examples">CVSS v3.1 Examples</a></li><li><a href="/cvss/v3.1/use-design">CVSS v3.1 Calculator Use & Design</a></li></ul></li><li><a href="/cvss/v3-0">CVSS v3.0 Archive</a><ul><li><a href="/cvss/calculator/3.0">CVSS v3.0 Calculator</a></li><li><a href="/cvss/v3.0/specification-document">CVSS v3.0 Specification Document</a></li><li><a href="/cvss/v3.0/user-guide">CVSS v3.0 User Guide</a></li><li><a href="/cvss/v3.0/examples">CVSS v3.0 Examples</a></li><li><a href="/cvss/v3.0/use-design">CVSS v3.0 Calculator Use & Design</a></li></ul></li><li><a href="/cvss/v2">CVSS v2 Archive</a><ul><li><a href="/cvss/v2/guide">CVSS v2 Complete Documentation</a></li><li><a href="/cvss/v2/history">CVSS v2 History</a></li><li><a href="/cvss/v2/team">CVSS-SIG team</a></li><li><a href="/cvss/v2/meetings">SIG Meetings</a></li><li><a href="/cvss/v2/faq">Frequently Asked Questions</a></li><li><a href="/cvss/v2/adopters">CVSS Adopters</a></li><li><a href="/cvss/v2/links">CVSS Links</a></li></ul></li><li><a href="/cvss/v1">CVSS v1 Archive</a><ul><li><a href="/cvss/v1/intro">Introduction to CVSS</a></li><li><a href="/cvss/v1/faq">Frequently Asked Questions</a></li><li><a href="/cvss/v1/guide">Complete CVSS v1 Guide</a></li></ul></li><li><a href="/cvss/data-representations">JSON & XML Data Representations</a></li><li><a href="/cvss/training">CVSS On-Line Training Course</a></li><li><a href="/cvss/identity">Identity & logo usage</a></li></ul></li><li><a href="/global/sigs/csirt">CSIRT Framework Development SIG</a></li><li><a href="/global/sigs/cyberinsurance">Cyber Insurance SIG</a><ul><li><a href="/global/sigs/cyberinsurance/events">Cyber Insurance SIG Webinars</a></li></ul></li><li><a href="/global/sigs/cti">Cyber Threat Intelligence SIG</a><ul><li><a href="/global/sigs/cti/curriculum/">Curriculum</a><ul><li><a href="/global/sigs/cti/curriculum/introduction">Introduction</a></li><li><a href="/global/sigs/cti/curriculum/cti-introduction">Introduction to CTI as a General topic</a></li><li><a href="/global/sigs/cti/curriculum/methods-methodology">Methods and Methodology</a></li><li><a href="/global/sigs/cti/curriculum/pir">Priority Intelligence Requirement (PIR)</a></li><li><a href="/global/sigs/cti/curriculum/source-evaluation">Source Evaluation and Information Reliability</a></li><li><a href="/global/sigs/cti/curriculum/machine-human">Machine and Human Analysis Techniques (and Intelligence Cycle)</a></li><li><a href="/global/sigs/cti/curriculum/threat-modelling">Threat Modelling</a></li><li><a href="/global/sigs/cti/curriculum/training">Training</a></li><li><a href="/global/sigs/cti/curriculum/standards">Standards</a></li><li><a href="/global/sigs/cti/curriculum/glossary">Glossary</a></li><li><a href="/global/sigs/cti/curriculum/cti-reporting/">Communicating Uncertainties in CTI Reporting</a></li></ul></li><li><a href="/global/sigs/cti/events/">Webinars and Online Training</a></li><li><a href="/global/sigs/cti/cti-program">Building a CTI program and team</a><ul><li><a href="/global/sigs/cti/cti-program/program-stages">Program maturity stages</a><ul><li><a href="/global/sigs/cti/cti-program/stage1">CTI Maturity model - Stage 1</a></li><li><a href="/global/sigs/cti/cti-program/stage2">CTI Maturity model - Stage 2</a></li><li><a href="/global/sigs/cti/cti-program/stage3">CTI Maturity model - Stage 3</a></li></ul></li><li><a href="/global/sigs/cti/cti-program/starter-kit">Program Starter Kit</a></li><li><a href="/global/sigs/cti/cti-program/resources">Resources and supporting materials</a></li></ul></li></ul></li><li><a href="/global/sigs/digital-safety">Digital Safety SIG</a></li><li><a href="/global/sigs/dns">DNS Abuse SIG</a><ul><li><a href="/global/sigs/dns/policies">Code of Conduct & Other Policies</a></li><li><a href="/global/sigs/dns/dns-abuse-examples">Examples of DNS Abuse</a></li></ul></li><li><a href="/global/sigs/ethics">Ethics SIG</a><ul><li><a href="/global/sigs/ethics/ethics-first">Ethics for Incident Response Teams</a></li></ul></li><li><a href="/epss/">Exploit Prediction Scoring System (EPSS)</a><ul><li><a href="/epss/model">The EPSS Model</a></li><li><a href="/epss/data_stats">Data and Statistics</a></li><li><a href="/epss/user-guide">User Guide</a></li><li><a href="/epss/research">EPSS Research and Presentations</a></li><li><a href="/epss/faq">Frequently Asked Questions</a></li><li><a href="/epss/who_is_using">Who is using EPSS?</a></li><li><a href="/epss/epss_tools">Open-source EPSS Tools</a></li><li><a href="/epss/api">API</a></li><li><a href="/epss/papers">Related Exploit Research</a></li><li><a>Blog</a><ul><li><a href="/epss/articles/prob_percentile_bins">Understanding EPSS Probabilities and Percentiles</a></li><li><a href="/epss/articles/log4shell">Log4Shell Use Case</a></li><li><a href="/epss/articles/estimating_old_cvss">Estimating CVSS v3 Scores for 100,000 Older Vulnerabilities</a></li></ul></li><li><a href="/epss/partners">Data Partners</a></li></ul></li><li><a href="/global/sigs/msr/">FIRST Multi-Stakeholder Ransomware SIG</a></li><li><a href="/global/sigs/hfs/">Human Factors in Security SIG</a></li><li><a href="/global/sigs/ics">Industrial Control Systems SIG (ICS-SIG)</a></li><li><a href="/global/sigs/iep">Information Exchange Policy SIG (IEP-SIG)</a></li><li><a href="/global/sigs/information-sharing">Information Sharing SIG</a><ul><li><a href="/global/sigs/information-sharing/misp">Malware Information Sharing Platform</a></li></ul></li><li><a href="/global/sigs/le">Law Enforcement SIG</a></li><li><a href="/global/sigs/malware">Malware Analysis SIG</a><ul><li><a href="/global/sigs/malware/ma-framework">Malware Analysis Framework</a></li><li><a href="/global/sigs/malware/ma-framework/malwaretools">Malware Analysis Tools</a></li></ul></li><li><a href="/global/sigs/metrics">Metrics SIG</a><ul><li><a href="/global/sigs/metrics/events">Metrics SIG Webinars</a></li></ul></li><li><a href="/global/sigs/netsec/">NETSEC SIG</a></li><li><a href="/global/sigs/passive-dns">Passive DNS Exchange</a></li><li><a href="/global/sigs/policy">Policy SIG</a></li><li><a href="/global/sigs/psirt">PSIRT SIG</a></li><li><a href="/global/sigs/red-team">Red Team SIG</a></li><li><a href="/global/sigs/cpg">Retail and Consumer Packaged Goods (CPG) SIG</a></li><li><a href="/global/sigs/ctf">Security Lounge SIG</a></li><li><a href="/global/sigs/tic/">Threat Intel Coalition SIG</a><ul><li><a href="/global/sigs/tic/membership-rules">Membership Requirements and Veto Rules</a></li></ul></li><li><a href="/global/sigs/tlp">Traffic Light Protocol (TLP-SIG)</a></li><li><a href="/global/sigs/transport">Transportation and Mobility SIG</a></li><li><a href="/global/sigs/vulnerability-coordination">Vulnerability Coordination</a><ul><li><a href="/global/sigs/vulnerability-coordination/multiparty">Multi-Party Vulnerability Coordination and Disclosure</a></li><li><a href="/global/sigs/vulnerability-coordination/multiparty/guidelines">Guidelines and Practices for Multi-Party Vulnerability Coordination and Disclosure</a></li></ul></li><li><a href="/global/sigs/vrdx">Vulnerability Reporting and Data eXchange SIG (VRDX-SIG)</a><ul><li><a href="/global/sigs/vrdx/vdb-catalog">Vulnerability Database Catalog</a></li></ul></li><li><a href="/global/sigs/wof">Women of FIRST</a></li></ul></li><li><a href="/global/governance">Internet Governance</a></li><li><a href="/global/irt-database">IR Database</a></li><li><a href="/global/fellowship">Fellowship Program</a><ul><li><a href="https://portal.first.org/fellowship">Application Form</a></li></ul></li><li><a href="/global/mentorship">Mentorship Program</a></li><li><a href="/hof">IR Hall of Fame</a><ul><li><a href="/hof/inductees">Hall of Fame Inductees</a></li></ul></li><li><a href="/global/victim-notification">Victim Notification</a></li><li><a href="/volunteers/">Volunteers at FIRST</a><ul><li><a href="/volunteers/list">FIRST Volunteers</a></li><li><a href="/volunteers/participation">Volunteer Contribution Record</a></li></ul></li><li><a href="#new">Previous Activities</a><ul><li><a href="/global/practices">Best Practices Contest</a></li></ul></li></ul></li><li><a href="/standards">Standards & Publications</a><ul><li><a href="/standards">Standards</a><ul><li><a href="/cvss">Common Vulnerability Scoring System (CVSS-SIG)</a></li><li><a href="/tlp">Traffic Light Protocol (TLP)</a><ul><li><a href="/tlp/use-cases">TLP Use Cases</a></li></ul></li><li><a href="/standards/frameworks/">Service Frameworks</a><ul><li><a href="/standards/frameworks/csirts">CSIRT Services Framework</a></li><li><a href="/standards/frameworks/psirts">PSIRT Services Framework</a></li></ul></li><li><a href="/iep">Information Exchange Policy (IEP)</a><ul><li><a href="/iep/iep_framework_2_0">IEP 2.0 Framework</a></li><li><a href="/iep/iep-json-2_0">IEP 2.0 JSON Specification</a></li><li><a href="/iep/iep-polices">Standard IEP Policies</a><ul><li><a href="https://www.first.org/iep/2.0/first-tlp-iep.iepj">IEP TLP Policy File</a></li><li><a href="https://www.first.org/iep/2.0/first-unknown-iep.iepj">IEP Unknown Policy File</a></li></ul></li><li><a href="/iep/iep_v1_0">IEP 1.0 Archive</a></li></ul></li><li><a href="/global/sigs/passive-dns">Passive DNS Exchange</a></li><li><a href="/epss">Exploit Prediction Scoring System (EPSS)</a></li></ul></li><li><a href="/resources/papers">Publications</a></li></ul></li><li><a href="/events">Events</a></li><li><a href="/education">Education</a><ul><li><a href="/education/first-training">FIRST Training</a><ul><li><a href="/education/trainings">Training Courses</a></li><li><a href="/education/trainers">FIRST Trainers</a></li></ul></li></ul></li><li><a href="/blog">Blog</a></li></ul> </div> </div> <div id="home-buttons"> <p><a href="/join" data-title="Join"><img alt="Join" src="/_/img/icon-join.svg"><span class="tt-join">Join<span>Details about FIRST membership and joining as a full member or liaison.</span></span></a> <a href="/learn" data-title="Learn"><img alt="Learn" src="/_/img/icon-learn.svg"><span class="tt-learn">Learn<span>Training and workshop opportunities, and details about the FIRST learning platform.</span></span></a> <a href="/participate" data-title="Participate"><img alt="Participate" src="/_/img/icon-participate.svg"><span class="tt-participate">Participate<span>Read about upcoming events, SIGs, and know what is going on.</span></span></a></p> </div></div></div></header><div id="body" data-studio="CU52CV1W8g"><div id="c1" data-studio="Yu8FjCC11g" class="toc-h2 toc-h3 image-center p"><h1 id="The-EPSS-Model">The EPSS Model</h1> <p>EPSS is a daily estimate of the probability of exploitation activity being observed over the next 30 days. It is designed from the ground up to make the best use of all of the information available and it does this in five steps:</p> <ol> <li>Collect as much vulnerability information as we can from a variety of sources</li> <li>Collect evidence of daily exploitation activity</li> <li>Train a model: discover/learn the relationship between the vulnerability information and the exploitation activity</li> <li>Measure the performance of the model, tweak and repeat step 3 to optimize the model</li> <li>On a daily basis: refresh the vulnerability information (step 1) and use the model (step 3) to produce daily estimates of the probability of exploitation in the next 30 days for each published CVE.</li> </ol> <p>We will walk through these steps below. For those wanting to know more, a detailed explanation of the background and rigor put into EPSS is covered in <a href="https://arxiv.org/abs/2302.14172">the latest published EPSS paper</a>.</p> <h2 id="Vulnerability-Information">Vulnerability Information</h2> <p>Collecting vulnerability information is all about gathering data that we hope will help us answer the question, "What makes a vulnerability more (or less) likely to be exploited?" Luckily, we don't have to know the answer, nor do we have to estimate any weights for the data we collect. The modeling in step 3 (in combination with the exploitation activity in step 2) will figure out how different sources of information help explain the exploitation activity we have observed. The more information we can collect the better, as more detail and variety in the data may help the model discover more and more subtle patterns about which vulnerabilities are likely to be exploited.</p> <p>The vulnerability information we collect:</p> <ul> <li>Vendor (CPE, via NVD)</li> <li>Age of the vulnerability (Days since CVE published in MITRE CVE list)</li> <li>References with categorical labels defining their content (MITRE CVE List, NVD)</li> <li>Normalized multiword expressions extracted from the description of the vulnerability (MITRE CVE List)</li> <li>Weakness in the vulnerability (CWE, via NVD)</li> <li>CVSS metrics (base vector from CVSS 3.x, via NVD)</li> <li>CVE is listed/discussed on a list or website (CISA KEV, Google Project Zero, Trend Micro's Zero Day Initiative (ZDI), with more being added)</li> <li>Publicly available exploit code (Exploit-DB, GitHub, MetaSploit)</li> <li>Offensive security tools and scanners: Intrigue, sn1per, jaeles, nuclei</li> </ul> <p>As EPSS evolves, this list will certainly be updated and expanded.</p> <p>Another important consideration within EPSS (and all of these data points) is that we are meticulous about collecting timing information. We want to know when information is published, added to lists or otherwise affects the threat landscape. It's not enough to know that a module was added to metasploit for a particular CVE, we need to know when it was added so we can look at the exploitation activity in the wild before and after the metasploit module was added. The attention to the timing applies to all of our data sources and it is especially important in the exploitation activity which we talk about in the next section.</p> <h2 id="Exploitation-Activity">Exploitation Activity</h2> <p>Feedback enables learning, which is why we are focused on gathering and organizing feedback in the form of exploitation activity in the wild from our <a href="https://www.first.org/epss/partners">data partners</a>. We are continuously working to expand the list of contributors - so if you work at or with any companies or vendors that have exploitation data and they aren't one of our contributors, ask them to contribute!</p> <p>Exploitation activity is evidence that exploitation of a vulnerability was attempted, not that it was successful against a vulnerable target. Which means we are collecting data from honeypots, IDS/IPS sensors and host-based detection methods and of course, we are always looking to expand data sources.</p> <p>We learned early on that exploitation activity is not a permanent stream of activity that once started, will continue indefinitely. Exploitation is bursty, often sporadic and sometimes isolated, localized and ephemeral. Getting a simple report that a vulnerability has "been exploited" in the wild doesn't help us understand exactly when, how often or how prevalent the exploitation activity has occurred. We need to know exactly when it occurred so we can measure if it was before or after specific events and we need to know if it is still being exploited. Without specific timing information we would struggle to accurately measure the effect of the various events we are collecting about each vulnerability.</p> <p>To highlight this point about timing, EPSS uses things like Google Project Zero and CISA KEV as vulnerability information and not as exploitation activity because their presence on the list is a single point in time (a vulnerability was added to a list) and always has occurred in the past (since EPSS is always looking forward to the next 30 days). By monitoring and collecting that information, we can build up an understanding of what happens after the vulnerability is added to the website or list. Perhaps listing on CISA's KEV list makes it less likely that attackers will use those CVEs moving forward since defenders may focus on remediating those before others. Maybe Google Project Zero is raising enough awareness that the zero-days end up less of a target once they reach the n-day status of a published CVE. Maybe both lists are raising awareness among the attackers and we may observe increased exploitation activity days or months after they were added to the list(s).</p> <p>Detailed exploitation activity along with the daily timing of that activity create the feedback loop that we leverage against the vulnerability information to train up a model.</p> <h2 id="EPSS-modeling">EPSS modeling</h2> <p>EPSS leverages machine learning to identify patterns and relationships between the vulnerability information and the exploitation activity that we have collected over time. We spend quite a bit of time on this step because we want to be sure the model is learning enough (performing well) but not too much (<a href="https://en.wikipedia.org/wiki/Overfitting">overfitting</a>). Model generation is tightly integrated with measuring the performance of the model. We will generate a model and measure its performance and repeat this over and over until we identify and maximize the effect of various permutations. For example, should we include all of the CWE's (there are nearly 600 unique CWE's used in NVD) or do we get a better model if we attempt to normalize the CWEs up to a specific view of parent nodes? What if we use two years of historical data instead of one year, or maybe six months? What if we tweak various model parameters on how fast or conservative the model learns? All of these questions can be answered by generating models and measuring the performance of that model (and repeating). Rather than focus on the modeling here, we will actually spend a lot more time focusing on measuring performance since that's what matters for using EPSS.</p> <h3 id="Predicting-the-Future">Predicting the Future</h3> <p>Measuring performance may seem impossible at first glance. After all, we are making predictions about the future and the future hasn't happened yet, right? How can we measure performance for something that hasn't happened yet? Recall that we are meticulous about the timing of things. Since we collect information about when information is published or when events occur, we can construct our "state of knowledge" at any point in time. This enables us to go back in time, reconstruct what we knew as of that point, even train a model and make predictions about "future" events (future according to the model, but it's in the past for us).</p> <p>Here's how it works in practice, EPSS is currently trained on 12 months worth of historical data. In order to test the performance on "future" data, we actually build those 12 months starting at 14 months ago and train the model on 12 months of data up to 2 months ago. This leaves us 2 months of "future" data that the model has never seen and knows nothing about. We can test all sorts of variations on models and data sources and we can see how well it will perform in the "future". We can even go back further and test how the model may degrade over time. Maybe the threat landscape shifts so often and so quickly that the model will be terrible in six months? These are all things we can (and have) tested. This approach and many other details about how we performed feature selection and engineering as well as model optimization are covered in <a href="https://arxiv.org/abs/2302.14172">our latest paper</a>.</p> <h3 id="A-Note-on-Prediction">A Note on Prediction</h3> <p>When most people hear "prediction" they may think of a mystic with a crystal ball who makes vaguely-specific statements about the future. Maybe you expect EPSS to tell you exactly what will and what won't be exploited and when. But that just isn't possible and anyone who makes that claim isn't being honest. One analogy often used is in gambling (probability theory was born from games of chance). Nobody can tell you which number will hit at the roulette table, nor predict which hands you will win at the blackjack table. But we can talk about the probability of each, and as that blackjack game unfolds we can update the probability with information observed on the table. This is why the output of EPSS is a probability. All we can say about vulnerability exploitation is that some of the vulnerabilities are more likely to be exploited than others, that's it, that's all EPSS is saying. EPSS is trying to provide information to practitioners that hopefully gives them an edge, that ability to play the odds and hopefully make proactive moves against attackers before they make those moves. Be careful not to compare any predictive system against perfection, instead compare against alternatives. While it'd be great to be perfect, in reality we need to identify the best strategy available to us and then try trusting it for a while.</p> <h2 id="Model-Performance">Model Performance</h2> <p>This is the good stuff. In order to measure performance we need two (perhaps obvious) things: our estimate of future events and feedback on whether or not those future events actually happened.</p> <p>Let's walk through an example and create a simple prioritization strategy (or model) that people are familiar with and prioritize all CVEs with a CVSS base score of 7 and above. We will talk later about measuring performance across a range of numbers, but for now, we will just make a binary choice of remediating CVSS score of 7 or above and delaying remediation for anything else and we will compare that against the outcome (observed exploitation activity). This gives each vulnerability two different attributes: if we prioritized or not, and if it had exploitation activity or not. Which means each vulnerability can be labeled with one of four categories, as shown in the figure below. To put numbers to this: As of October 1st, 2023, NVD has published 139,473 CVSS 3.x scores for published CVEs. In the following 30 days (Oct 1 - Oct 30th), we observed 3,852 unique CVEs (with CVSS 3.x scores) with exploitation activity (about 2.7%). Note that we saw a lot more CVEs exploited than that in October, but we are measuring CVSS here and so we limit to those with CVSS 3.x scores. For those 139-thousand we can construct a 2x2 table with the two variables (left), and we can visualize the proportions with a venn diagram (right).</p> <img src="images/cvss7_truth_matrix.png" width="80%" /> <p>The large gray circle represents all published CVEs with an NVD-assigned CVSS 3.x score, and all of the CVEs that were scored with 7 or higher with CVSS are shown in blue (our "strategy" says we should prioritize these). And finally all of the CVEs that we observed as being exploited in the following 30 days are shown in red (these are the ones we should have prioritized).</p> <ul> <li><strong>True positives (TP)</strong> are the good decisions -- the prioritized vulnerabilities that we also observed exploitation activity against in the wild. This is where the blue circle of prioritized vulnerabilities overlaps with the red circle of exploited vulnerabilities.</li> <li><strong>False positives (FP)</strong> are vulnerabilities that were prioritized, but not exploited. These decisions represent potentially wasted resources and are in the blue circle of prioritized vulnerabilities that did not overlap with the red circle of exploited vulnerabilities.</li> <li><strong>False negatives (FN)</strong> are vulnerabilities that were not prioritized but were observed to be exploited in the wild. These are the vulnerabilities in the red circle not being overlapped by the remediated vulnerabilities in the blue circle.</li> <li><strong>True Negatives (TN</strong>) are vulnerabilities not prioritized and not exploited, these are the vulnerabilities in the outer gray circle that were neither remediated nor exploited in the wild.</li> </ul> <p>As the figure above shows, the strategy to remediate based on CVSS 7+ prioritizes a large portion of those published CVEs (blue part overlapping the gray) and produces many false positives (the blue part not overlapping the red) and still leaves many of the exploited vulnerabilities open and waiting to be remediated (the red not overlapped by the blue).</p> <p>Let's compare this to EPSS with a completely arbitrary cutoff set at 10% (Note we say "arbitrary" here because 10% is not an official EPSS threshold, nor any sort of statement about where you should set a threshold. You can use 10% if you'd like, but just know we neither endorse nor oppose this as a threshold and here it is an arbitrary number that we happen to chose for this example.)</p> <img src="images/epss_10pct_truth_matrix.png" width="80%" /> <p>The first obvious difference between the two is the size of the blue circle, or the amount of effort each strategy would dictate. With an EPSS threshold set at 10%, the effort is greatly reduced. We also see the False Positives greatly reduced, which is a good thing, but it appears to be at the expense of some True Positives.</p> <p>Looking at the numbers in the tables is nice but perhaps a little confusing, four intertwined and connected variables is a little much to really determine value and/or performance of any given strategy. We can simplify this table down to two numbers that most organizations will want to track over time: <strong>efficiency</strong> and <strong>coverage</strong> and of course we also want to keep an eye on <strong>effort</strong>, since that is heavily tied to staffing, resources and budgets.</p> <h3 id="Effort-Efficiency-and-Coverage">Effort, Efficiency and Coverage</h3> <p>Using these four categories (TP, FP, FN, TN) we can derive three more meaningful metrics, what we've termed as effort, efficiency and coverage. The last two, efficiency and coverage, are also referred to as <a href="https://en.wikipedia.org/wiki/Precision_and_recall">precision and recall</a> respectively.</p> <p><strong>Effort</strong> is measuring the proportion of vulnerabilities being prioritized. <a href="https://www.cyentia.com/pithy-p2p/">Research by Cyentia and Kenna Security</a> (now part of Cisco) has shown that most organizations are able to remediate an average of 10-15% of their open vulnerabilities per month, this is something to keep in mind when comparing different strategies and the time/resources they would demand of your stakeholders.</p> <p><strong>Efficiency</strong> considers how efficiently resources were spent by measuring the percent of prioritized vulnerabilities that were exploited. In the above diagram, efficiency is the amount of the blue circle covered by the red circle. Prioritizing mostly exploited vulnerabilities would be a high efficiency rating (resources were allocated efficiently), while prioritizing perhaps random or mostly non-exploited vulnerabilities would result in a low efficiency rating. Efficiency is calculated as the number of exploited vulnerabilities prioritized (TP) divided by the total number of prioritized vulnerabilities (TP+FP).</p> <p><strong>Coverage</strong> considers how well is the percent of exploited vulnerabilities that were prioritized, and is calculated as the number of exploited vulnerabilities prioritized (TP) divided by the total number of exploited vulnerabilities (TP + FN). In the above diagram, coverage is the amount of the red circle covered by the blue circle. Having low coverage indicates that not many of the exploited vulnerabilities were remediated with the given strategy.</p> <p>These three variables are intertwined. Within a single strategy, getting better coverage comes with more effort and lower efficiency, while improving efficiency often lowers effort and coverage. That trade off can be broken by finding an improved prioritization strategy, which is why it can be so important to track these metrics and compare across different strategies.</p> <p>We can return to the examples above and calculate effort, efficiency and coverage. The effort for a strategy that remediates CVSS 7 and above is 57.4% (<code>(tp + fp) / (tp + fp + tn + fn) or 80024 / 139473</code>), coverage is 82.2% (<code>tp / (tp + fn) or 3166 / (3166 + 686)</code>) and the efficiency is 3.96% (<code>tp / (tp + fp) or 3166 / (3166 + 76858)</code>). For the EPSS strategy of 0.1 and above we have an effort of 2.7% (<code>3735/139473</code>), coverage of 63.2% (<code>2435 / 3852</code>) and efficiency of 65.2% (<code>2435 / 3735</code>). Visually this is shown with the two venn diagrams next to each other:</p> <img src="images/venn_with_text_overlay.png" width="70%" /> <h3 id="Measuring-Performance-Across-all-Possible-Scores">Measuring Performance Across all Possible Scores</h3> <p>EPSS has never published guidance around thresholds that everyone should use. There is no universal "critical" or "high" that we could agree on. Hopefully breaking out and understanding measurements like <em>effort</em>, <em>efficiency</em> and <em>coverage</em> helps explain why. Those thresholds represent statements of risk tolerance and different organizations will obviously approach vulnerability prioritization differently based on their internal risk tolerance and resource constraints. Firms that do not have many resources (staff/budget) may wish to emphasize efficiency (at the expense of coverage) to limit their effort to get the best impact from the limited resources available. But for firms where resources are less constrained and security leans more towards mission critical, the emphasis can be on getting higher coverage at the expense of both effort and efficiency.</p> <p>With that in mind, thresholds are commonly used and sought after, so we can provide guidance about the ranges of effort, coverage and efficiency we may expect at different thresholds. Hopefully this will help you and your organization have an informed discussion around which thresholds may be right for you, your staff and your budget constraints.</p> <p>A line plot is often described as a "point in motion". Given a range of possible thresholds, we can sequentially measure coverage and efficiency (and effort) at each threshold, put a point on a scatter plot and connect those points with a line. It should help us understand and visualize the tradeoff between coverage and efficiency at various thresholds.</p> <img src="images/october_PR_curves.png" width="80%" /> <p>Looking at the above plot may be a bit confusing at first, but focus in on CVSS 7 and EPSS 0.1 (10%). We already know a threshold set at CVSS 7 and above has a coverage and efficiency of 82.2% and 4% respectively, and that's where that particular point is in the horizontal (coverage) and vertical (efficiency). Same is true for an EPSS threshold at 10%, we calculated the coverage and efficiency to be 63.2% and 65.2% respectively and that's where that particular point is on the EPSS line. Looking at this plot, it should help you set your own thresholds. Maybe you have a limited budget and the most "critical" in your environment is set to EPSS 50% and above, providing coverage around 45% and efficiency around 85%. Maybe your organization has a much lower risk tolerance (and more budget) and your "critical" should be a much lower threshold, say 5% or maybe even 1%, the choice is all yours!</p></div></div><div id="navbar" data-studio="CU52CV1W8g"><div id="c4" data-studio="Yu8FjCC11g"><ul class="navbar"><li><a href="/epss">Exploit Prediction Scoring System (EPSS)</a><ul><li><a href="/epss/model">The EPSS Model</a></li><li><a href="/epss/data_stats">Data and Statistics</a></li><li><a href="/epss/user-guide">User Guide</a></li><li><a href="/epss/research">EPSS Research and Presentations</a></li><li><a href="/epss/faq">Frequently Asked Questions</a></li><li><a href="/epss/who_is_using">Who is using EPSS?</a></li><li><a href="/epss/epss_tools">Open-source EPSS Tools</a></li><li><a href="/epss/api">API</a></li><li><a href="/epss/papers">Related Exploit Research</a></li><li><a>Blog</a><ul><li><a href="/epss/articles/prob_percentile_bins">Understanding EPSS Probabilities and Percentiles</a></li><li><a href="/epss/articles/log4shell">Log4Shell Use Case</a></li><li><a href="/epss/articles/estimating_old_cvss">Estimating CVSS v3 Scores for 100,000 Older Vulnerabilities</a></li></ul></li><li><a href="/epss/partners">Data Partners</a></li></ul></li></ul></div></div><div id="sidebar" data-studio="CU52CV1W8g"></div><footer><div id="footer" data-studio="CU52CV1W8g"><div id="c2" data-studio="Yu8FjCC11g"><div class="content"> <div class="support"> <div class="kbsearch bottom"> <p><a href="https://support.first.org"><img src="/_/img/icon-portal_support.svg" alt="FIRST Support" title="FIRST Support" /></a> <input class="kb-search" type="search" placeholder="Do you need help?"></p> </div> </div> <div id="socialnetworks"><a href="/about/sdg" title="FIRST Supported Sustainable Development Goals (SDG)" class="icon-sdg"></a><a rel="me" href="https://infosec.exchange/@firstdotorg" target="_blank" title="@FIRSTdotOrg@infosec.exchange" class="icon-mastodon"></a><a href="https://twitter.com/FIRSTdotOrg" target="_blank" title="Twitter @FIRSTdotOrg" class="icon-tw"></a><a href="https://www.linkedin.com/company/firstdotorg" target="_blank" title="FIRST.Org at LinkedIn" class="icon-linkedin"></a><a href="https://www.facebook.com/FIRSTdotorg" target="_blank" title="FIRST.Org at Facebook" class="icon-fb"></a><a href="https://github.com/FIRSTdotorg" target="_blank" title="FIRST.Org at Github" class="icon-github"></a><a href="https://www.youtube.com/c/FIRSTdotorg" target="_blank" title="FIRST.Org at Youtube" class="icon-youtube"></a><a href="/podcasts" title="FIRST.Org Podcasts" class="icon-podcast"></a></div> <p><a href="/copyright">Copyright</a> © 2015—2024 by Forum of Incident Response and Security Teams, Inc. All Rights Reserved.</p> </div> <p><span class="tlp"></span></p></div></div></footer><script nonce="uFCK4JBvDwBa7S1BleCkMw" async="async" src="/_/web.js?20241125212614"></script><script nonce="uFCK4JBvDwBa7S1BleCkMw" async="async" src="/_/s.js?20241125-212616"></script></body></html>