CINXE.COM
AI Red Teaming: Applying Software TEVV for AI Evaluations | CISA
<!DOCTYPE html> <html lang="en" dir="ltr" prefix="og: https://ogp.me/ns#" class="no-js"> <head> <meta charset="utf-8" /> <link rel="canonical" href="https://www.cisa.gov/news-events/news/ai-red-teaming-applying-software-tevv-ai-evaluations" /> <meta property="og:site_name" content="Cybersecurity and Infrastructure Security Agency CISA" /> <meta property="og:type" content="website" /> <meta property="og:url" content="https://www.cisa.gov/news-events/news/ai-red-teaming-applying-software-tevv-ai-evaluations" /> <meta property="og:title" content="AI Red Teaming: Applying Software TEVV for AI Evaluations | CISA" /> <meta name="Generator" content="Drupal 10 (https://www.drupal.org)" /> <meta name="MobileOptimized" content="width" /> <meta name="HandheldFriendly" content="true" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <link rel="icon" href="/profiles/cisad8_gov/themes/custom/gesso/favicon.png" type="image/png" /> <title>AI Red Teaming: Applying Software TEVV for AI Evaluations | CISA</title> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/align.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/fieldgroup.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/container-inline.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/clearfix.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/details.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/hidden.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/item-list.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/js.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/nowrap.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/position-container.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/reset-appearance.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/resize.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/system-status-counter.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/system-status-report-counters.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/system-status-report-general-info.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/system/css/components/tablesort.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/misc/components/progress.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/misc/components/ajax-progress.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/core/modules/views/css/views.module.css?snxaec" /> <link rel="stylesheet" media="all" href="/profiles/cisad8_gov/modules/custom/toolbar_tasks/css/toolbar.css?snxaec" /> <link rel="stylesheet" media="all" href="/modules/contrib/extlink/css/extlink.css?snxaec" /> <link rel="stylesheet" media="all" href="/modules/contrib/ckeditor_accordion/css/accordion.frontend.css?snxaec" /> <link rel="stylesheet" media="all" href="/modules/contrib/paragraphs/css/paragraphs.unpublished.css?snxaec" /> <link rel="stylesheet" media="all" href="/modules/contrib/better_social_sharing_buttons/css/better_social_sharing_buttons.css?snxaec" /> <link rel="stylesheet" media="all" href="//fonts.googleapis.com/css2?family=Montserrat:wght@400;500;600;700&family=Public+Sans:wght@400;500;600;700&display=swap" /> <link rel="stylesheet" media="all" href="/profiles/cisad8_gov/themes/custom/gesso/dist/css/styles.css?snxaec" /> <script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"node\/22630","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"gtm":{"tagId":null,"settings":{"data_layer":"dataLayer","include_classes":false,"allowlist_classes":"","blocklist_classes":"","include_environment":false,"environment_id":"","environment_token":""},"tagIds":["GTM-53QLXSL9"]},"gtag":{"tagId":"","consentMode":false,"otherIds":[],"events":[],"additionalConfigInfo":[]},"ajaxPageState":{"libraries":"eJxdjdFuwzAIRX_IjT_JwoY6XoiJALfN38_q2k7byxXnCjiZ3EmTSWnAyVbQ1mvKw126RfOTJ4eyETYXTVCKKDbp8TMtV5Xu1DHQw-f2FlHHAby8MFSRypQcaqwz_vMCX_D4W-6hDrujxcqSgd_kBEYaDlCoCsdqb9Nvs4x-jMzNVsJgpzntMc-r4CKcQafAtvnph8KwK2GVW3rpoQOf3opFFsDLBy8Fe7g1ult85rILDqZvwyd8uQ","theme":"guswds","theme_token":null},"ajaxTrustedUrl":[],"data":{"extlink":{"extTarget":false,"extTargetAppendNewWindowLabel":"(opens in a new window)","extTargetNoOverride":false,"extNofollow":false,"extNoreferrer":false,"extFollowNoOverride":false,"extClass":"ext","extLabel":"(link is external)","extImgClass":false,"extSubdomains":true,"extExclude":"(.\\.gov$)|(.\\.mil$)|(.\\.mil\/)|(.\\.gov\/)","extInclude":"","extCssExclude":".c-menu--social,.c-menu--footer,.c-social-links,.c-text-cta--button,.usa-footer__contact-info","extCssInclude":"","extCssExplicit":"","extAlert":true,"extAlertText":"You are now leaving an official website of the United State Government (USG), the Department of Homeland Security (DHS) and the Cybersecurity and Infrastructure Security Agency (CISA). Links to non-USG, non-DHS and non-CISA sites are provided for the visitor\u0027s convenience and do not represent an endorsement by USG, DHS or CISA of any commercial or private issues, products or services. Note that the privacy policy of the linked site may differ from that of USG, DHS and CISA.","extHideIcons":false,"mailtoClass":"mailto","telClass":"","mailtoLabel":"(link sends email)","telLabel":"(link is a phone number)","extUseFontAwesome":false,"extIconPlacement":"append","extFaLinkClasses":"fa fa-external-link","extFaMailtoClasses":"fa fa-envelope-o","extAdditionalLinkClasses":"","extAdditionalMailtoClasses":"","extAdditionalTelClasses":"","extFaTelClasses":"fa fa-phone","whitelistedDomains":[],"extExcludeNoreferrer":""}},"ckeditorAccordion":{"accordionStyle":{"collapseAll":1,"keepRowsOpen":0,"animateAccordionOpenAndClose":1,"openTabsWithHash":1}},"user":{"uid":0,"permissionsHash":"0f75d40308887aebba0d5b0d2671305b73c9431902f86e672380a6dc6ab97d07"}}</script> <script src="/core/assets/vendor/jquery/jquery.min.js?v=3.7.1"></script> <script src="/core/assets/vendor/once/once.min.js?v=1.0.1"></script> <script src="/core/misc/drupalSettingsLoader.js?v=10.3.6"></script> <script src="/core/misc/drupal.js?v=10.3.6"></script> <script src="/core/misc/drupal.init.js?v=10.3.6"></script> <script src="/core/assets/vendor/tabbable/index.umd.min.js?v=6.2.0"></script> <script src="/modules/contrib/google_tag/js/gtm.js?snxaec"></script> <script src="/modules/contrib/google_tag/js/gtag.js?snxaec"></script> <script src="/core/misc/progress.js?v=10.3.6"></script> <script src="/core/assets/vendor/loadjs/loadjs.min.js?v=4.3.0"></script> <script src="/core/misc/debounce.js?v=10.3.6"></script> <script src="/core/misc/announce.js?v=10.3.6"></script> <script src="/core/misc/message.js?v=10.3.6"></script> <script src="/core/misc/ajax.js?v=10.3.6"></script> <script src="/modules/contrib/google_tag/js/gtag.ajax.js?snxaec"></script> </head> <body class="path-node not-front node-page node-page--node-type-article" id="top"> <div class="c-skiplinks"> <a href="#main" class="c-skiplinks__link u-visually-hidden u-focusable">Skip to main content</a> </div> <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-53QLXSL9" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas> <div class="l-site-container"> <section class="usa-banner" aria-label="Official government website"> <div class="usa-accordion"> <header class="usa-banner__header"> <div class="usa-banner__inner"> <div class="grid-col-auto"> <img class="usa-banner__header-flag" src="/profiles/cisad8_gov/themes/custom/gesso/dist/images/us_flag_small.png" alt="U.S. flag" /> </div> <div class="grid-col-fill tablet:grid-col-auto"> <p class="usa-banner__header-text">An official website of the United States government</p> <p class="usa-banner__header-action" aria-hidden="true">Here’s how you know</p></div> <button class="usa-accordion__button usa-banner__button" aria-expanded="false" aria-controls="gov-banner"> <span class="usa-banner__button-text">Here’s how you know</span> </button> </div> </header> <div class="usa-banner__content usa-accordion__content" id="gov-banner"> <div class="grid-row grid-gap-lg"> <div class="usa-banner__guidance tablet:grid-col-6"> <img class="usa-banner__icon usa-media-block__img" src="/profiles/cisad8_gov/themes/custom/gesso/dist/images/icon-dot-gov.svg" alt="Dot gov"> <div class="usa-media-block__body"> <p> <strong>Official websites use .gov</strong> <br> A <strong>.gov</strong> website belongs to an official government organization in the United States. </p> </div> </div> <div class="usa-banner__guidance tablet:grid-col-6"> <img class="usa-banner__icon usa-media-block__img" src="/profiles/cisad8_gov/themes/custom/gesso/dist/images/icon-https.svg" alt="HTTPS"> <div class="usa-media-block__body"> <p> <strong>Secure .gov websites use HTTPS</strong> <br> A <strong>lock</strong> (<span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" width="52" height="64" viewBox="0 0 52 64" class="usa-banner__lock-image" role="img" aria-labelledby="banner-lock-title banner-lock-description"><title id="banner-lock-title">Lock</title><desc id="banner-lock-description">A locked padlock</desc><path fill="#000000" fill-rule="evenodd" d="M26 0c10.493 0 19 8.507 19 19v9h3a4 4 0 0 1 4 4v28a4 4 0 0 1-4 4H4a4 4 0 0 1-4-4V32a4 4 0 0 1 4-4h3v-9C7 8.507 15.507 0 26 0zm0 8c-5.979 0-10.843 4.77-10.996 10.712L15 19v9h22v-9c0-6.075-4.925-11-11-11z"/></svg></span>) or <strong>https://</strong> means you’ve safely connected to the .gov website. Share sensitive information only on official, secure websites. </p> </div> </div> </div> </div> </div> </section> <div class="c-block c-global-header-btns c-global-btns"> <div class="l-constrain l-constrain"> <div class="c-block__content"> <div id="block-globalbuttons" class="c-block c-block--provider-block-content c-block--id-block-content83069f9f-34fc-4d54-86ec-936a204f8088"> <div class="c-block__content"> <div class="c-field c-field--name-body c-field--type-text-with-summary c-field--label-hidden"> <div class="c-field__content"><p><a class="c-button c-button--basic c-button--blue" href="/resources-tools/resources/free-cybersecurity-services-and-tools" title="Free Cyber Services">Free Cyber Services</a><a class="c-button c-button--basic c-button--green60" href="/topics/election-security/election-threat-updates">Election Threat Updates</a><a class="c-button c-button--basic c-button--gray" href="/protect2024">#protect2024</a><a class="c-button c-button--basic c-button--teal" href="/node/18883">Secure Our World</a><a class="c-button c-button--campaign" href="/node/8056">Shields Up</a><a class="c-button c-button--report" href="/report">Report A Cyber Issue</a></p></div></div> </div> </div> </div> </div> </div> <div class="usa-overlay"></div> <header class="usa-header usa-header--extended" role="banner"> <div class="usa-navbar"> <div class="l-constrain"> <div class="usa-navbar__row"> <div class="c-block c-site-header"> <div class="l-constrain"> <div class="c-block__content"> <div id="block-guswds-cisaheaderblock" class="c-block c-block--provider-block-content c-block--id-block-contentbc4e6844-86b4-4e20-b163-a73bda3d1d76"> <div class="c-block__content"> <div class="c-field c-field--name-body c-field--type-text-with-summary c-field--label-hidden"> <div class="c-field__content"><a href="/"><img src = "/sites/default/files/images/SVG/header_logo_tagline_update.svg" alt="CISA logo image. America's Cyber Defense Agency, National Coordinator for Critical Infrastructure Security and Resilience"/></a></div></div> </div> </div> </div> </div> </div> <div class="c-block c-site-header-mobile"> <div class="l-constrain"> <div class="c-block__content"> <div id="block-guswds-cisaheaderblockmobile" class="c-block c-block--provider-block-content c-block--id-block-content283396c9-cd36-4ce3-b1e2-9b5576ab4f50"> <div class="c-block__content"> <div class="c-field c-field--name-body c-field--type-text-with-summary c-field--label-hidden"> <div class="c-field__content"><a href="/"><img src = "/sites/default/files/images/SVG/mobile_logo_wordmark.svg" alt="CISA Logo"/></a></div></div> </div> </div> </div> </div> </div> <div class="usa-navbar__search"> <div class="usa-navbar__search-header"> <p>Search</p> </div> <div class="usa-search"> <script async src=https://cse.google.com/cse.js?cx=ffc4c79e29d5b3a8c></script> <div class="gcse-searchbox-only" data-resultsurl="/search"> </div> </div> </div> <button class="mobile-menu-button usa-menu-btn">Menu</button> </div> </div> </div> <div class="c-block c-tagline-mobile"> <div class="l-constrain"> <div class="c-block__content"> <div id="block-guswds-mobiletaglinecontainer" class="c-block c-block--provider-block-content c-block--id-block-contentc8d12e9d-7e48-4708-90c1-563609c4b566"> <div class="c-block__content"> <div class="c-field c-field--name-body c-field--type-text-with-summary c-field--label-hidden"> <div class="c-field__content"><p><center><img src = "/sites/default/files/images/SVG/header_tagline_mobile_update.svg" alt = "America's Cyber Defense Agency" /></center></div></div> </div> </div> </div> </div> </div> <nav class="usa-nav" role="navigation" aria-label="Primary navigation"> <div class="usa-nav__inner l-constrain"> <div class="usa-nav__row"> <button class="usa-nav__close">Close</button> <div class="usa-search"> <script async src=https://cse.google.com/cse.js?cx=ffc4c79e29d5b3a8c></script> <div class="gcse-searchbox-only" data-resultsurl="/search"> </div> </div> <ul class="usa-nav__primary usa-accordion"> <li class="usa-nav__primary-item topics"> <button class="usa-accordion__button usa-nav__link " aria-expanded="false" aria-controls="basic-mega-nav-section-1"> <span>Topics</span> </button> <div id="basic-mega-nav-section-1" class="usa-nav__submenu usa-megamenu" hidden=""> <div class="usa-megamenu__parent-link"> <a href="/topics">Topics</a> </div> <div class="usa-megamenu__menu-items"> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/cybersecurity-best-practices"> <span>Cybersecurity Best Practices</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/cyber-threats-and-advisories"> <span>Cyber Threats and Advisories</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/critical-infrastructure-security-and-resilience"> <span>Critical Infrastructure Security and Resilience</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/election-security"> <span>Election Security</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/emergency-communications"> <span>Emergency Communications</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/industrial-control-systems"> <span>Industrial Control Systems</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/information-communications-technology-supply-chain-security"> <span>Information and Communications Technology Supply Chain Security</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/partnerships-and-collaboration"> <span>Partnerships and Collaboration</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/physical-security"> <span>Physical Security</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/topics/risk-management"> <span>Risk Management</span> </a> </div> </div> </div> <div class="c-menu-feature-links"> <div class="c-menu-feature-links__title"> <a href="/audiences"> How can we help? </a> </div> <div class="c-menu-feature-links__content"><a href="/topics/government">Government</a><a href="/topics/educational-institutions">Educational Institutions</a><a href="/topics/industry">Industry</a><a href="/topics/state-local-tribal-and-territorial">State, Local, Tribal, and Territorial</a><a href="/topics/individuals-and-families">Individuals and Families</a><a href="/topics/small-and-medium-businesses">Small and Medium Businesses</a><a href="/audiences/find-help-locally">Find Help Locally</a><a href="/audiences/faith-based-community">Faith-Based Community</a><a href="/audiences/executives">Executives</a><a href="/audiences/high-risk-communities">High-Risk Communities</a></div> </div> </div> </li> <li class="usa-nav__primary-item spotlight"> <a href="/spotlight" class="usa-nav__link" > <span>Spotlight</span> </a> </li> <li class="usa-nav__primary-item resources--tools"> <button class="usa-accordion__button usa-nav__link " aria-expanded="false" aria-controls="basic-mega-nav-section-3"> <span>Resources & Tools</span> </button> <div id="basic-mega-nav-section-3" class="usa-nav__submenu usa-megamenu" hidden=""> <div class="usa-megamenu__parent-link"> <a href="/resources-tools">Resources & Tools</a> </div> <div class="usa-megamenu__menu-items"> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/resources-tools/all-resources-tools"> <span>All Resources & Tools</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/resources-tools/services"> <span>Services</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/resources-tools/programs"> <span>Programs</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/resources-tools/resources"> <span>Resources</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/resources-tools/training"> <span>Training</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/resources-tools/groups"> <span>Groups</span> </a> </div> </div> </div> </div> </li> <li class="usa-nav__primary-item news--events"> <button class="usa-accordion__button usa-nav__link usa-current" aria-expanded="false" aria-controls="basic-mega-nav-section-4"> <span>News & Events</span> </button> <div id="basic-mega-nav-section-4" class="usa-nav__submenu usa-megamenu" hidden=""> <div class="usa-megamenu__parent-link"> <a href="/news-events">News & Events</a> </div> <div class="usa-megamenu__menu-items"> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/news-events/news"> <span>News</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/news-events/events"> <span>Events</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/news-events/cybersecurity-advisories"> <span>Cybersecurity Alerts & Advisories</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/news-events/directives"> <span>Directives</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/news-events/request-speaker"> <span>Request a CISA Speaker</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/news-events/congressional-testimony"> <span>Congressional Testimony</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/cisa-conferences"> <span>CISA Conferences</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/cisa-live"> <span>CISA Live!</span> </a> </div> </div> </div> </div> </li> <li class="usa-nav__primary-item careers"> <button class="usa-accordion__button usa-nav__link " aria-expanded="false" aria-controls="basic-mega-nav-section-5"> <span>Careers</span> </button> <div id="basic-mega-nav-section-5" class="usa-nav__submenu usa-megamenu" hidden=""> <div class="usa-megamenu__parent-link"> <a href="/careers">Careers</a> </div> <div class="usa-megamenu__menu-items"> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/careers/benefits-perks"> <span>Benefits & Perks</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/careers/hirevue-applicant-reasonable-accommodations-process"> <span>HireVue Applicant Reasonable Accommodations Process</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/general-recruitment-and-hiring-faqs"> <span>Hiring</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/careers/resume-application-tips"> <span>Resume & Application Tips</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/students-recent-graduates-employment-opportunities"> <span>Students & Recent Graduates</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/careers/veteran-and-military-spouse-employment-opportunities"> <span>Veteran and Military Spouses</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/careers/work-cisa"> <span>Work @ CISA</span> </a> </div> </div> </div> </div> </li> <li class="usa-nav__primary-item about"> <button class="usa-accordion__button usa-nav__link " aria-expanded="false" aria-controls="basic-mega-nav-section-6"> <span>About</span> </button> <div id="basic-mega-nav-section-6" class="usa-nav__submenu usa-megamenu" hidden=""> <div class="usa-megamenu__parent-link"> <a href="/about">About</a> </div> <div class="usa-megamenu__menu-items"> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/culture"> <span>Culture</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/divisions-offices"> <span>Divisions & Offices</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/regions"> <span>Regions</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/leadership"> <span>Leadership</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/doing-business-cisa"> <span>Doing Business with CISA</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/site-links"> <span>Site Links</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/reporting-employee-and-contractor-misconduct"> <span>Reporting Employee and Contractor Misconduct</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/cisa-github"> <span>CISA GitHub</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/cisa-central"> <span>CISA Central</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/2023YIR"> <span>2023 Year In Review</span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/contact-us"> <span>Contact Us </span> </a> </div> </div> <div class="usa-col"> <div class="usa-nav__submenu-item"> <a href="/about/contact-us/subscribe-updates-cisa"> <span>Subscribe</span> </a> </div> </div> </div> </div> </li> </ul> <div class="c-block c-global-menu-btns c-global-btns"> <div class="c-block__content"> <div id="block-globalbuttons" class="c-block c-block--provider-block-content c-block--id-block-content83069f9f-34fc-4d54-86ec-936a204f8088"> <div class="c-block__content"> <div class="c-field c-field--name-body c-field--type-text-with-summary c-field--label-hidden"> <div class="c-field__content"><p><a class="c-button c-button--basic c-button--blue" href="/resources-tools/resources/free-cybersecurity-services-and-tools" title="Free Cyber Services">Free Cyber Services</a><a class="c-button c-button--basic c-button--green60" href="/topics/election-security/election-threat-updates">Election Threat Updates</a><a class="c-button c-button--basic c-button--gray" href="/protect2024">#protect2024</a><a class="c-button c-button--basic c-button--teal" href="/node/18883">Secure Our World</a><a class="c-button c-button--campaign" href="/node/8056">Shields Up</a><a class="c-button c-button--report" href="/report">Report A Cyber Issue</a></p></div></div> </div> </div> </div> </div> </div> </div> </nav> </header> <div class="l-breadcrumb"> <div class="l-constrain"> <div class="l-breadcrumb__row"> <nav aria-labelledby="breadcrumb-label" class="c-breadcrumb" role="navigation"> <div class="l-constrain"> <div id="breadcrumb-label" class="c-breadcrumb__title u-visually-hidden">Breadcrumb</div> <ol class="c-breadcrumb__list"> <li class="c-breadcrumb__item"> <a class="c-breadcrumb__link" href="/">Home</a> </li> <li class="c-breadcrumb__item"> <a class="c-breadcrumb__link" href="/news-events">News & Events</a> </li> <li class="c-breadcrumb__item"> <a class="c-breadcrumb__link" href="/news-events/news">News</a> </li> </ol> </div> </nav> <div id="block-bettersocialsharingbuttons" class="c-block c-block--social-share c-block--provider-better-social-sharing-buttons c-block--id-social-sharing-buttons-block"> <div class="c-block__content"> <div class="c-block__row"> <span>Share:</span> <div style="display: none"><link rel="preload" href="/modules/contrib/better_social_sharing_buttons/assets/dist/sprites/social-icons--no-color.svg" as="image" type="image/svg+xml" crossorigin="anonymous" /></div> <div class="social-sharing-buttons"> <a href="https://www.facebook.com/sharer/sharer.php?u=https://www.cisa.gov/news-events/news/ai-red-teaming-applying-software-tevv-ai-evaluations&title=AI%20Red%20Teaming%3A%20Applying%20Software%20TEVV%20for%20AI%20Evaluations" target="_blank" title="Share to Facebook" aria-label="Share to Facebook" class="social-sharing-buttons__button share-facebook" rel="noopener"> <svg width="18px" height="18px" style="border-radius:3px;"> <use href="/modules/contrib/better_social_sharing_buttons/assets/dist/sprites/social-icons--no-color.svg#facebook" /> </svg> </a> <a href="https://twitter.com/intent/tweet?text=AI%20Red%20Teaming%3A%20Applying%20Software%20TEVV%20for%20AI%20Evaluations+https://www.cisa.gov/news-events/news/ai-red-teaming-applying-software-tevv-ai-evaluations" target="_blank" title="Share to X" aria-label="Share to X" class="social-sharing-buttons__button share-x" rel="noopener"> <svg width="18px" height="18px" style="border-radius:3px;"> <use href="/modules/contrib/better_social_sharing_buttons/assets/dist/sprites/social-icons--no-color.svg#x" /> </svg> </a> <a href="https://www.linkedin.com/sharing/share-offsite/?url=https://www.cisa.gov/news-events/news/ai-red-teaming-applying-software-tevv-ai-evaluations" target="_blank" title="Share to Linkedin" aria-label="Share to Linkedin" class="social-sharing-buttons__button share-linkedin" rel="noopener"> <svg width="18px" height="18px" style="border-radius:3px;"> <use href="/modules/contrib/better_social_sharing_buttons/assets/dist/sprites/social-icons--no-color.svg#linkedin" /> </svg> </a> <a href="mailto:?subject=AI%20Red%20Teaming%3A%20Applying%20Software%20TEVV%20for%20AI%20Evaluations&body=https://www.cisa.gov/news-events/news/ai-red-teaming-applying-software-tevv-ai-evaluations" title="Share to Email" aria-label="Share to Email" class="social-sharing-buttons__button share-email" target="_blank" rel="noopener"> <svg width="18px" height="18px" style="border-radius:3px;"> <use href="/modules/contrib/better_social_sharing_buttons/assets/dist/sprites/social-icons--no-color.svg#email" /> </svg> </a> </div> </div> </div> </div> </div> </div> </div> <main id="main" class="c-main" role="main" tabindex="-1"> <div class="l-content"> <div class="is-promoted l-full"> <div class="l-full__header"> <div class="c-page-title"> <div class="c-page-title__inner l-constrain"> <div class="c-page-title__row"> <div class="c-page-title__content"> <div class="c-page-title__meta">Blog</div> <h1 class="c-page-title__title"> <span>AI Red Teaming: Applying Software TEVV for AI Evaluations</span> </h1> <div class="c-page-title__date"> <div class="c-field c-field--name-field-release-date c-field--type-datetime c-field--label-above"> <div class="c-field__label">Released</div><div class="c-field__content"><time datetime="2024-11-26T12:00:00Z">November 26, 2024</time></div></div> </div> <div class="c-page-title__date"></div> <div class="c-page-title__author"> <p>By: Jonathan Spring, Deputy Chief AI Officer and Divjot Singh Bawa, Strategic Advisor</p> </div> <div class="c-page-title__topic"> <div class="c-topic__label"> Related topics: </div> <div class="c-top__topics"> <a href="/topics/cybersecurity-best-practices">Cybersecurity Best Practices</a>, <a href="/topics/cyber-threats-and-advisories">Cyber Threats and Advisories</a> </div> </div> </div> </div> <div class="c-page-title__decoration"></div> </div> </div> </div> <div class="l-full__main"> <div class="c-wysiwyg"> <div class="l-constrain"> <div class="c-wysiwyg__inner"> <div class="c-field c-field--name-field-full-html c-field--type-text-long c-field--label-hidden"> <div class="c-field__content"><p>As the <a href="/national-security-memorandum-critical-infrastructure-security-and-resilience" title="National Security Memorandum on Critical Infrastructure Security and Resilience" data-entity-type="node" data-entity-uuid="62e5936d-9fb2-4061-8a54-34c13436000d" data-entity-substitution="canonical">National Coordinator</a> for critical infrastructure security and resilience, CISA is <a href="/resources-tools/resources/roadmap-ai" title="Roadmap for AI" data-entity-type="node" data-entity-uuid="14994f98-5b3b-46c0-8735-4b4dadf5c23b" data-entity-substitution="canonical">responsible</a> for facilitating a <a href="/securebydesign" title="Secure by Design" data-entity-type="node" data-entity-uuid="d9d42dfb-bf8b-48c3-b1be-f58c015a6dec" data-entity-substitution="canonical">Secure by Design</a> approach to <a href="/news-events/news/software-must-be-secure-design-and-artificial-intelligence-no-exception" title="Software Must Be Secure by Design, and Artificial Intelligence Is No Exception" data-entity-type="node" data-entity-uuid="fbd725a1-754a-40df-925e-915f0c319304" data-entity-substitution="canonical">AI-based software</a> across the digital ecosystem and helping protect critical infrastructure from malicious uses of AI. To effectively mitigate against critical failures, physical attacks, and cyberattacks, AI software developers must prioritize conducting rigorous safety and security testing to understand how an AI system can fail or be exploited.</p><p>AI red teaming is a foundational component of the safety and security evaluations process. </p><p>This blogpost demonstrates that AI red teaming must fit into the existing framework for AI Testing, Evaluation, Validation and Verification (TEVV). Additionally, the post explains how and why AI TEVV must fit into software TEVV, ensuring AI systems are <a href="https://csrc.nist.gov/glossary/term/fit_for_purpose" title="NIST Fit for Purpose">fit for purpose</a>. While there are differences in the specific software tools used, AI TEVV—despite common misconceptions—must be treated under software TEVV from a strategic and operational perspective.</p><p>This assertion is grounded in the fact that TEVV has been used for more than four decades to improve the safety and security of software.<sup>1</sup> Experts working on AI evaluations should avoid reinventing the wheel and build upon lessons the software security community has learned through developing and improving guidance and requirements.</p><h2>Framing AI Red Teaming in the Context of TEVV</h2><p><a href="https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/" title="Executive Order on the Safe, Secure, and Trustworthy Development and Use of Artificial Intelligence">AI red teaming</a> is the third-party safety and security evaluation of AI systems; AI red teaming is a subset of AI Testing, Evaluation, Verification and Validation (TEVV).</p><p><a href="https://www.nist.gov/ai-test-evaluation-validation-and-verification-tevv" title="NIST AI Test, Evaluation, Validation and Verification (TEVV)">AI TEVV</a>, a broader risk-based approach for the external testing of AI systems, has been developed and operationalized by our interagency partners at the National Institute of Standards (NIST) through programs like <a href="https://www.nist.gov/ai-test-evaluation-validation-and-verification-tevv" title="NIST AI Test, Evaluation, Validation and Verification (TEVV)">Assessing Risks and Impacts of AI (ARIA)</a> and the <a href="https://ai-challenges.nist.gov/genai" title="NIST Evaluating Generative AI Technologies">NIST GenAI Challenge</a>.</p><p>Because AI systems are a <a href="/news-events/news/software-must-be-secure-design-and-artificial-intelligence-no-exception" title="Software Must Be Secure by Design, and Artificial Intelligence Is No Exception" data-entity-type="node" data-entity-uuid="fbd725a1-754a-40df-925e-915f0c319304" data-entity-substitution="canonical">type of software system</a>, approaches for AI TEVV must fundamentally be a sub-component of the more established software TEVV.<sup>2</sup> The TEVV framework is commonly used to test software reliability and help ensure that software is <a href="https://csrc.nist.gov/glossary/term/fit_for_purpose" title="NIST Fit for Purpose">fit for purpose</a>. TEVV can broadly be deconstructed into three components: <a href="https://csrc.nist.gov/glossary/term/software_system_test_and_evaluation_process" title="NIST software system test and evaluation process">software system test and evaluation process</a>, <a href="https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-160v1r1.pdf" title="NIST Engineering Trustworthy Secure Systems">software verification</a>, and <a href="https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-160v1r1.pdf" title="NIST Engineering Trustworthy Secure Systems">software validation.</a></p><h2>Software TEVV Can be Used for AI Evaluations</h2><p>A common misconception surrounding AI evaluation methods is that the established software TEVV framework is not, or cannot be, adapted to account for the evaluation of AI systems.</p><p>However, while there are tactical implementation and technical details that differ between AI TEVV and software TEVV, the two processes—from a strategic and operational vantage point—are quite similar. There are three truths about all software systems that illuminate this assertion:</p><h3>1. Software systems have always had safety risks </h3><p>One pervasive example that fuels the misconception that AI and software TEVV are dissimilar is the narrative that AI evaluations are unique because of the need to mitigate risk posed by potential security vulnerabilities and safety violations.<sup>3</sup> While true, many software developers have long had to consider both the security and safety dimensions within traditional software systems.</p><p>For example, the Food and Drug Administration (FDA) approves medical devices for use within the United States. Since the 1980s, some medical devices have had a software component, a trend that is increasingly common today. In 1986, software flaws in a cancer radiotherapy device, the Therac-25, led to several deaths.<sup>4</sup> The software flaw was a <a href="https://en.wikipedia.org/wiki/Race_condition" title="Wikipedia Race Condition">race condition</a>, a type of error where operations are not executed in the correct order, resulting in unexpected outcomes. They occur often unpredictably and arise due to complex interactions between components and data. Race conditions are often hard to reproduce and identifying which single lines of code require modification to fix the flaw can be challenging.</p><p>While the FDA’s device approval process has since been <a href="https://www.fda.gov/medical-devices/digital-health-center-excellence/cybersecurity" title="FDA Cybersecurity">updated</a>, the Therac-25 incident demonstrates how traditional software can have fatal consequences for human safety. Medical devices are not uniquely susceptible to software safety risks; many other critical infrastructure sectors also employ safety critical software. Examples include <a href="https://sma.nasa.gov/sma-disciplines/software-assurance-and-software-safety" title="NASA Software Assurance and Software Safety">aerospace</a>, <a href="https://www.epa.gov/climate-change-water-sector/water-and-wastewater-systems-sector-cybersecurity" title="EPA Water and Wastewater Systems Sector Cybersecurity">water/wastewater</a>, and <a href="https://www.transportation.gov/sites/dot.gov/files/2024-04/HASS_COE_AI_Assurance_Whitepaper_Apr2024.pdf" title="An Overview of AI Assurance for Transportation">transportation</a>, among many others. AI systems, as one type of software system, should similarly be evaluated for safety concerns, cybersecurity vulnerabilities, and in particular cybersecurity issues that could be exploited to cause safety issues.</p><h3>2. Software systems require validity and reliability testing</h3><p>Another common misconception is that AI systems must distinctively be tested for <a href="https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf" title="NIST Artificial Intelligence Risk Management Framework">validity and reliability</a>, preventing the deployment of AI systems which are inaccurate, unreliable, or poorly generalized to data.</p><p>However, mitigating against validity and reliability concerns while also ensuring the robustness of software against novel situations and inputs is common to both software and AI. For example, modern road vehicle braking systems often heavily rely on software to work effectively. Automated braking software interprets data from sensors and assists when a driver may not react a hazard in time.</p><p>This safety-critical software must demonstrate robustness to a variety of events and conditions, like unexpected pedestrians, slick roads, or the driver following too closely behind another car. Designers of all safety-critical software systems, whether they include an AI-element or not, must consider a range of factors including the dynamics of the system, the probability of certain external events, and the desired degree of “safety margin” given the impact of the system losing control. Additionally, rigorous testing is applied to ensure that the real system reflects the intended design assumptions.</p><p>While AI systems can be more complex than this simple example, many of the concepts and techniques from evaluating and modeling software robustness against unexpected inputs are akin to those used in traditional software evaluations.</p><h3>3. Software systems are fundamentally probabilistic</h3><p>Finally, many point to how AI systems, constructed with probabilities and commonly created with intentional variance to avoid producing repetitive results, often need multiple trials to discover improper behavior. This concern surrounding variability also extends to how outputs from AI systems may differ entirely even with small changes to configuration details or training data.</p><p>However, traditional software systems are also inherently unpredictable and can exhibit wildly different behavior based on small changes in input if appropriate safeguards are not implemented. Broadly, one cannot prove any non-trivial properties of any computer program in general ahead of time (i.e., <a href="https://en.wikipedia.org/wiki/Rice%27s_theorem" title="Wikipedia Rice's Theorem">Rice’s Theorem</a>).</p><p>However, more operationally relevant are security vulnerabilities where a change to one or a few bytes in the input from the network can lead to total control of the machine by a threat actor; this happened with the popular web server NGINX in 2021 (<a href="https://nvd.nist.gov/vuln/detail/CVE-2021-23017" title="NIST CVE-2021-23017 Detail">CVE-2021-23017</a>). Some classes of vulnerabilities, like race conditions, are not deterministic in any software system. Software engineers may also intentionally create controlled randomness in computer processes; this is a core function of cryptography.</p><p>Characteristics that may seem more prominent or concerning in AI systems (safety concerns, testing for validity and reliability, their probabilistic nature) have always been present in traditional software systems. As such, the well-established software TEVV methodology is a perfectly valid approach from which to conduct AI evaluations. Yes, there are differences with AI that require some adaptation, but none so large as to warrant a drastically different approach.</p><h2>CISA's Role in AI TEVV</h2><p>As the AI evaluations field continues to mature, there is an array of diverse stakeholders working to advance the science and practice of AI red teaming. This includes developing novel methodologies, creating tools that are interoperable across models or platforms, and improving capabilities to conduct AI red teaming at scale.</p><p>Serving both as <a href="/national-security-memorandum-critical-infrastructure-security-and-resilience" title="National Security Memorandum on Critical Infrastructure Security and Resilience" data-entity-type="node" data-entity-uuid="62e5936d-9fb2-4061-8a54-34c13436000d" data-entity-substitution="canonical">National Coordinator</a> and an operational lead for federal cybersecurity, CISA focuses on contributing to AI red teaming efforts that primarily support security evaluations for Federal and non-Federal entities; this work is organized into three broad workstreams.</p><p>First, CISA remains steadfast in ensuring that our work on AI pre-deployment testing supplements efforts in industry, academia, and government. CISA is a founding member of the recently <a href="https://www.commerce.gov/news/press-releases/2024/11/us-ai-safety-institute-establishes-new-us-government-taskforce#:~:text=The%20Testing%20Risks%20of%20AI%20for%20National%20Security,concerns%20and%20strengthen%20American%20leadership%20in%20AI%20innovation." title="U.S. AI Safety Institute Establishes New U.S. Government Taskforce to Collaborate on Research and Testing of AI Models to Manage National Security Capabilities & Risks">announced </a>Testing Risks of AI for National Security (TRAINS) Taskforce, which will include testing of advanced AI models across national security and public safety domains. Led by the NIST AI Safety Institute, CISA will contribute expertise both by helping build new AI evaluation methods and benchmarks that integrate with security testing processes, as well as providing subject matter expertise on cybersecurity testing. For much of this work, CISA will rely upon Vulnerability Management within CISA’s Cybersecurity Division, which offers CISA security evaluation services such as <a href="/cyber-hygiene-services" title="Cyber Hygiene Services" data-entity-type="node" data-entity-uuid="72160b5e-4761-4fdb-9c92-286ca4365ef1" data-entity-substitution="canonical">Cyber Hygiene</a> and <a href="/sites/default/files/publications/VM_Assessments_Fact_Sheet_RVA_508C.pdf" title="Cyber Assessment Fact Sheet">Risk and Vulnerability Assessments</a>.</p><p>Second, CISA continues to provide technical assistance and risk management support to Federal and non-Federal partners, specifically supporting AI security technical post-deployment testing. This includes varied forms of testing, such as <a href="/sites/default/files/publications/VM_Assessments_Fact_Sheet_RVA_508C.pdf" title="Cyber Assessment Fact Sheet">penetration testing</a>, <a href="/cyber-hygiene-services" title="Cyber Hygiene Services" data-entity-type="node" data-entity-uuid="72160b5e-4761-4fdb-9c92-286ca4365ef1" data-entity-substitution="canonical">vulnerability scanning</a>, and <a href="/sites/default/files/publications/VM_Assessments_Fact_Sheet_RVA_508C.pdf" title="Cyber Assessment Fact Sheet">configuration testing</a>. CISA also often works independently to detect and identify security vulnerabilities impacting critical infrastructure systems and devices. CISA has already begun to receive requests from partners to conduct penetration and technical security testing on Large Language Models (LLMs) and expects demand for these services to grow as partners increasingly adopt AI tools.</p><p>Third, CISA collaborates with NIST on the development of standards for AI security testing. CISA provides operational cybersecurity expertise to help make standards practicable. Additionally, CISA builds on NIST standards to provide high-quality services and advice to our partners. CISA security evaluation services, such as red teaming, include AI systems in the scope of those security assessment services. CISA also provides operational guidance for securing software systems, such as the <a href="/cybersecurity-performance-goals" title="Cybersecurity Performance Goals" data-entity-type="node" data-entity-uuid="cfae1f93-c367-423f-886c-7db7684215a3" data-entity-substitution="canonical">cross-sector cybersecurity performance goals</a>, and priority security practices for AI systems, such as the Secure by Design <a href="/securebydesign/pledge" title="Secure by Design Pledge" data-entity-type="node" data-entity-uuid="d0a12de6-6e1a-4728-aacc-beb9e9824b5b" data-entity-substitution="canonical">pledge</a>.</p><h2>New, but the Same</h2><p>By treating AI TEVV as a subset of traditional software TEVV, the AI evaluations community benefits from using and building upon decades of proven and tested approaches towards assuring software is fit for purpose. Additionally, by streamlining processes, enterprises can avoid standing up parallel testing processes to accomplish similar ends, saving time and resources.</p><p>Most notably, with the knowledge that software and AI TEVV must be treated similarly to software TEVV from a strategic and operational perspective, the digital ecosystem can instead channel effort at the tactical level, developing novel tools, applications, and benchmarks to robustly execute AI TEVV.</p><p> </p><hr><p><sup>1</sup> The US Department of Defense publications in the Rainbow Series in the 1980s represent major development in the history of cybersecurity guidance and requirements. See <a href="https://irp.fas.org/nsa/rainbow.htm" title="NSA/NCSC Rainbow Series">NSA/NCSC Rainbow Series</a>.</p><p><sup>2</sup> Some organizations approach software TEVV as a part of a broader product quality management regime, as defined by the processes and practices necessary for conformance with the ISO 9000 series of publications. For example, NIST SP 800-160r1 applies ISO 9000 definitions for validation and verification directly to the topic of engineering trustworthy, secure information systems. However, NIST SP 800-160r1 refers to a different ISO standard (29119-2:2021 – Software and systems engineering – Software testing) to define testing. There are several other documents referenced by SP 800-160r1 for evaluation criteria and processes. While SP 800-160r1 does not title itself a TEVV process manual, it does in fact define and inter-relate the processes for software testing, evaluation, validation, and verification.</p><p><sup>3</sup> NIST defines “security” as resistance to intentional, unauthorized act(s) designed to cause harm or damage to a system; “safety” is a property of a system such that it does not, under defined conditions, lead to a state in which human life, health, property, or the environment is endangered. See: <a href="https://airc.nist.gov/AI_RMF_Knowledge_Base/Glossary" title="NIST The Language of Trustworthy AI: An In-Depth Glossary of Terms">The Language of Trustworthy AI: An In-Depth Glossary of Terms</a>.</p><p><sup>4</sup> Leveson, Nancy G., and Clark S. Turner. "An investigation of the Therac-25 accidents." Computer 26.7 (1993): 18-41.c</p></div></div> </div> </div> </div> </div> <div class="l-full__footer"> <div class="c-view c-view--detail-page-related-content c-view--display-block_1 view js-view-dom-id-4711082e55be65109260a0a0aac03fb3855892b1ed83b64fe24cdeb12eaba187 c-collection c-collection--blue c-collection--two-column"> <div class="l-constrain"> <div class="c-collection__row"> <div class="c-collection__content"> <h2 class="c-collection__title"><span class="c-collection__title-wrap">Related Articles</span></h2> </div> <div class="c-collection__cards"> <article class="is-promoted c-teaser c-teaser--horizontal" role="article"> <div class="c-teaser__row"> <div class="c-teaser__content"> <div class="c-teaser__eyebrow"> <div class="c-teaser__date"><time datetime="2024-05-02T12:00:00Z">May 02, 2024</time> </div> <div class="c-teaser__meta">Blog</div> </div> <h3 class="c-teaser__title"> <a href="/news-events/news/under-digital-radar-defending-against-peoples-republic-chinas-nation-state-cyber-threats-americas" target="_self"> <span>Under the Digital Radar: Defending Against People’s Republic of China’s Nation-State Cyber Threats to America’s Small Businesses</span> </a> </h3> </div> </div> </article> <article class="is-promoted c-teaser c-teaser--horizontal" role="article"> <div class="c-teaser__row"> <div class="c-teaser__content"> <div class="c-teaser__eyebrow"> <div class="c-teaser__date"><time datetime="2023-11-28T12:00:00Z">Nov 28, 2023</time> </div> <div class="c-teaser__meta">Blog</div> </div> <h3 class="c-teaser__title"> <a href="/news-events/news/unlocking-tomorrows-cybersecurity-sneak-peek-readysetcyber" target="_self"> <span>Unlocking Tomorrow’s Cybersecurity: A Sneak Peek into ReadySetCyber</span> </a> </h3> </div> </div> </article> <article class="is-promoted c-teaser c-teaser--horizontal" role="article"> <div class="c-teaser__row"> <div class="c-teaser__content"> <div class="c-teaser__eyebrow"> <div class="c-teaser__date"><time datetime="2023-09-29T12:00:00Z">Sep 29, 2023</time> </div> <div class="c-teaser__meta">Blog</div> </div> <h3 class="c-teaser__title"> <a href="/news-events/news/transforming-vulnerability-management-cisa-adds-oasis-csaf-20-standard-ics-advisories" target="_self"> <span>Transforming Vulnerability Management: CISA Adds OASIS CSAF 2.0 Standard to ICS Advisories </span> </a> </h3> </div> </div> </article> <article class="is-promoted c-teaser c-teaser--horizontal" role="article"> <div class="c-teaser__row"> <div class="c-teaser__content"> <div class="c-teaser__eyebrow"> <div class="c-teaser__date"><time datetime="2023-09-18T12:00:00Z">Sep 18, 2023</time> </div> <div class="c-teaser__meta">Blog</div> </div> <h3 class="c-teaser__title"> <a href="/news-events/news/kev-catalog-reaches-1000-what-does-mean-and-what-have-we-learned" target="_self"> <span>KEV Catalog Reaches 1000, What Does That Mean and What Have We Learned </span> </a> </h3> </div> </div> </article> </div> </div> </div> </div> </div> </div> </div> </main> <footer class="usa-footer usa-footer--slim" role="contentinfo"> <div class="usa-footer__return-to-top"> <div class="l-constrain"> <a href="#">Return to top</a> </div> </div> <div class="usa-footer__upper"> <div class="l-constrain"> <ul class="c-menu c-menu--footer-main"> <li class="c-menu__item"> <a href="/topics" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/7329">Topics</a> </li> <li class="c-menu__item"> <a href="/spotlight" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/7330">Spotlight</a> </li> <li class="c-menu__item"> <a href="/resources-tools" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/7331">Resources & Tools</a> </li> <li class="c-menu__item is-active-trail"> <a href="/news-events" class="c-menu__link js-top-level is-active-trail" aria-current="false" data-drupal-link-system-path="node/7332">News & Events</a> </li> <li class="c-menu__item"> <a href="/careers" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/7323">Careers</a> </li> <li class="c-menu__item"> <a href="/about" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/6944">About</a> </li> </ul> </div> </div> <div class="usa-footer__main"> <div class="l-constrain"> <div class="usa-footer__main-row"> <div class="usa-footer__brand"> <a class="c-site-name c-site-name--footer" href="/" rel="home" title="Go to the Cybersecurity & Infrastructure Security Agency homepage"> <span class="c-site-name__text">Cybersecurity & Infrastructure Security Agency</span> </a> </div> <div class="usa-footer__contact"> <ul class="c-menu c-menu--social"> <li class="c-menu__item"> <a href="https://www.facebook.com/CISA" class="c-menu__link--facebook c-menu__link js-top-level" aria-current="false">Facebook</a> </li> <li class="c-menu__item"> <a href="https://twitter.com/CISAgov" class="c-menu__link--twitter c-menu__link js-top-level" aria-current="false">Twitter</a> </li> <li class="c-menu__item"> <a href="https://www.linkedin.com/company/cybersecurity-and-infrastructure-security-agency" class="c-menu__link--linkedin c-menu__link js-top-level" aria-current="false">LinkedIn</a> </li> <li class="c-menu__item"> <a href="https://www.youtube.com/@cisagov" class="c-menu__link--youtube c-menu__link js-top-level" aria-current="false">YouTube</a> </li> <li class="c-menu__item"> <a href="https://www.instagram.com/cisagov" class="c-menu__link--instagram c-menu__link js-top-level" aria-current="false">Instagram</a> </li> <li class="c-menu__item"> <a href="/subscribe-updates-cisa" class="c-menu__link--rss c-menu__link js-top-level" aria-current="false">RSS</a> </li> </ul> <div class="usa-footer__contact-info"> <span>CISA Central</span> <a href="tel:1-844-Say-CISA">1-844-Say-CISA</a> <a href="mailto:SayCISA@cisa.dhs.gov">SayCISA@cisa.dhs.gov</a> </div> </div> </div> </div> </div> <div class="usa-footer__lower"> <div class="l-constrain"> <div class="usa-footer__lower-row"> <div class="usa-footer__lower-left"> <div class="c-dhs-logo"> <div class="c-dhs-logo__seal">DHS Seal</div> <div class="c-dhs-logo__content"> <div class="c-dhs-logo__url">CISA.gov</div> <div class="c-dhs-logo__text">An official website of the U.S. Department of Homeland Security</div> </div> </div> <ul class="c-menu c-menu--footer"> <li class="c-menu__item"> <a href="/about" class="c-menu__link js-top-level" title="About CISA" aria-current="false" data-drupal-link-system-path="node/6944">About CISA</a> </li> <li class="c-menu__item"> <a href="https://www.dhs.gov/performance-financial-reports" class="c-menu__link js-top-level" title="Budget and Performance" aria-current="false">Budget and Performance</a> </li> <li class="c-menu__item"> <a href="https://www.dhs.gov" title="Department of Homeland Security" class="c-menu__link js-top-level" aria-current="false">DHS.gov</a> </li> <li class="c-menu__item"> <a href="/oedia" title="Equal Opportunity & Accessibility" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/21462">Equal Opportunity & Accessibility</a> </li> <li class="c-menu__item"> <a href="https://www.dhs.gov/foia" class="c-menu__link js-top-level" title="FOIA Requests" aria-current="false">FOIA Requests</a> </li> <li class="c-menu__item"> <a href="/no-fear-act" title="No FEAR Act Reporting" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="node/21494">No FEAR Act</a> </li> <li class="c-menu__item"> <a href="https://www.oig.dhs.gov/" class="c-menu__link js-top-level" title="Office of Inspector General" aria-current="false">Office of Inspector General</a> </li> <li class="c-menu__item"> <a href="/privacy-policy" class="c-menu__link js-top-level" title="Privacy Policy" aria-current="false" data-drupal-link-system-path="node/16115">Privacy Policy</a> </li> <li class="c-menu__item"> <a href="https://public.govdelivery.com/accounts/USDHSCISA/subscriber/new?topic_id=USDHSCISA_138" title="Subscribe to Email Updates" class="c-menu__link js-top-level" aria-current="false">Subscribe</a> </li> <li class="c-menu__item"> <a href="https://www.whitehouse.gov/" class="c-menu__link js-top-level" title="The White House" aria-current="false">The White House</a> </li> <li class="c-menu__item"> <a href="https://www.usa.gov/" class="c-menu__link js-top-level" title="USA.gov" aria-current="false">USA.gov</a> </li> <li class="c-menu__item"> <a href="/forms/feedback" title="Website Feedback" class="c-menu__link js-top-level" aria-current="false" data-drupal-link-system-path="forms/feedback">Website Feedback</a> </li> </ul> </div> <div class="usa-footer__lower-right"> <iframe src="https://www.dhs.gov/ntas/" name="National Terrorism Advisory System" title="National Terrorism Advisory System" width="170" height="180" scrolling="no" frameborder="0" seamless border="0" ></iframe> </div> </div> </div> </div> </footer> </div> </div> <script src="/profiles/cisad8_gov/themes/custom/gesso/dist/js/common.js?snxaec"></script> <script src="/profiles/cisad8_gov/themes/custom/gesso/dist/js/uswds-init.es6.js?snxaec"></script> <script src="/profiles/cisad8_gov/themes/custom/gesso/dist/js/uswds.es6.js?snxaec"></script> <script src="https://dap.digitalgov.gov/Universal-Federated-Analytics-Min.js?agency=DHS&subagency=CISA&yt=true" id="_fed_an_ua_tag"></script> <script src="/modules/contrib/extlink/js/extlink.js?v=10.3.6"></script> <script src="/modules/contrib/ckeditor_accordion/js/accordion.frontend.min.js?snxaec"></script> <script src="/profiles/cisad8_gov/themes/custom/gesso/dist/js/teaser.es6.js?snxaec"></script> </body> </html>