CINXE.COM
Supporting benchmarks for AI safety with MLCommons
<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> <meta name="description" content="Posted by Anoop Sinha, Technology and Society, and Marian Croak, Google Research, Responsible AI and Human Centered Technology team Standard benchm..."><meta name="keywords" content="AI,Collaboration,Responsible AI"><link rel="canonical" href="https://research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" /><meta property="og:title" content="Supporting benchmarks for AI safety with MLCommons"><meta property="og:url" content="https://research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/"><meta property="og:description" content="Posted by Anoop Sinha, Technology and Society, and Marian Croak, Google Research, Responsible AI and Human Centered Technology team Standard benchm..."><meta property="og:image" content="https://storage.googleapis.com/gweb-research2023-media/images/bd1f9a0da60ef93b224d8e4a484f4860-G.width-800.format-jpeg.jpg"><meta property="og:image:secure_url" content="https://storage.googleapis.com/gweb-research2023-media/images/bd1f9a0da60ef93b224d8e4a484f4860-G.width-800.format-jpeg.jpg"><meta property="og:type" content="Website"> <title>Supporting benchmarks for AI safety with MLCommons</title> <meta name="description" content="Posted by Anoop Sinha, Technology and Society, and Marian Croak, Google Research, Responsible AI and Human Centered Technology team Standard benchm..." /> <meta name="viewport" content="width=device-width, initial-scale=1 viewport-fit=cover"/> <link rel="icon" type="image/png" href="/gr/static/assets/favicon.ico"> <link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link rel="preload" href="https://fonts.googleapis.com/css2?family=Product+Sans&family=Google+Sans+Display:ital@0;1&family=Google+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&family=Google+Sans+Text:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&display=swap" as="style"> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Product+Sans&family=Google+Sans+Display:ital@0;1&family=Google+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&family=Google+Sans+Text:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&display=swap"> <link href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@400;700&display=swap" rel="stylesheet"> <link href="https://www.gstatic.com/glue/cookienotificationbar/cookienotificationbar.min.css" rel="stylesheet" /> <link href="https://www.gstatic.com/glue/v27_1/glue-material.min.css" rel="stylesheet"> <link rel="stylesheet" type="text/css" href="/gr/static/css/googleresearch.css?id=0c26ea1fed8bdd0324f9f4fad1f6a470"> <script> window.dataLayer = window.dataLayer || []; dataLayer.push({ publishDate: '20231026', wordCount: '1074' }); </script> <!-- Google Tag Manager --> <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-K8QBZ7Q'); </script> <!-- End Google Tag Manager --> </head> <body class=" js-google-tag-wrapper" data-gt-page-path="https://research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" data-env="production"> <!-- Google Tag Manager (noscript) --> <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-K8QBZ7Q" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <!-- End Google Tag Manager (noscript) --> <header class="global-header glue-header glue-header--single not-glue"> <a href="#page-content" class="glue-header__skip-content">Jump to Content</a> <div class="glue-header__bar glue-header__bar--mobile not-glue"> <div class="glue-header__tier not-glue"> <!-- mobile lockup component --> <div class="glue-header__container"> <div class="glue-header__lock-up"> <!-- Hamburger button component --> <div class="glue-header__hamburger"> <button class="glue-header__drawer-toggle-btn" aria-label="Open the navigation drawer"> <svg class="glue-icon glue-icon--24px" role="presentation" aria-hidden="true"> <use href="/gr/static/assets/icons/glue-icons.svg#menu"></use> </svg> </button> </div> <div class="glue-header__logo"> <a class="glue-header__logo-link" href="/" title="Google Research"> <!-- Logo component --> <div class="glue-header__logo-container"> <svg role="presentation" aria-hidden="true" alt='Google' class="glue-icon glue-icon glue-header__logo-svg"> <use href="/gr/static/assets/icons/glue-icons.svg#google-color-logo"></use> </svg> </div> <span class="glue-header__logo--product">Research</span> </a> </div> </div> </div> </div> </div> <div class="glue-header__bar glue-header__bar--desktop glue-header__drawer"> <div class="glue-header__tier"> <!-- desktop lockup component --> <div class="glue-header__container"> <div class="glue-header__lock-up"> <div class="glue-header__logo"> <a class="glue-header__logo-link" href="/" title="Google Research"> <!-- Logo component --> <div class="glue-header__logo-container"> <svg role="presentation" aria-hidden="true" alt='Google' class="glue-icon glue-icon glue-header__logo-svg not-glue --dark-logo"> <use href="/gr/static/assets/icons/glue-icons.svg#google-solid-logo"></use> </svg> <svg role="presentation" aria-hidden="true" alt='Google' class="glue-icon glue-icon glue-header__logo-svg --light-logo"> <use href="/gr/static/assets/icons/glue-icons.svg#google-color-logo"></use> </svg> </div> <span class="glue-header__logo--product">Research</span> </a> </div> </div> </div> <!-- linkbar component --> <div class="glue-header__container glue-header__container--linkbar"> <nav class="glue-header__link-bar navigation js-gt-global-nav-wrapper"> <ul class="glue-header__list"> <li class="glue-header__item js-sub-nav-parent --parent" data-gt-primary="Who we are" > <button class="glue-header__link js-sub-nav-target" aria-haspopup="true" aria-expanded="false" > <span class=""> Who we are <span class="icon icon--caret"></span> </span> </button> <div class="navigation__sub js-sub-nav" role="menu"> <div class="navigation__sub__container"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Who we are <span class="sr-text">menu</span> </button> <hr/> </div> <div class="block-nav_drawer_columns_content"> <div class="navigation__sub--content" data-gt-secondary="Defining the technology of today and tomorrow."> <div class="navigation__sub__wrapper"> <div class="navigation__sub__heading"> <h2 class="headline-3">Defining the technology of today and tomorrow.</h2> </div> <ul class="navigation__sub__columns"> <li data-gt-secondary="Philosophy"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Philosophy </h2> <p class="navigation__sub__columns__description caption">We strive to create an environment conducive to many different types of research across many different time scales and levels of risk.</p> <a href="https://research.google/philosophy/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Philosophy</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/philosophy/" > Philosophy </a> </div> </li> <li data-gt-secondary="People"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> People </h2> <p class="navigation__sub__columns__description caption">Our researchers drive advancements in computer science through both fundamental and applied research.</p> <a href="https://research.google/people/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our People</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/people/" > People </a> </div> </li> </ul> </div> </div> </div> </div> </div> </li> <li class="glue-header__item js-sub-nav-parent --parent" data-gt-primary="Research areas" > <button class="glue-header__link js-sub-nav-target" aria-haspopup="true" aria-expanded="false" > <span class=""> Research areas <span class="icon icon--caret"></span> </span> </button> <div class="navigation__sub js-sub-nav" role="menu"> <div class="navigation__sub__container"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Research areas <span class="sr-text">menu</span> </button> <hr/> </div> <div class="block-nav_drawer_columns_link_list"> <div class="navigation__sub--list"> <div class="navigation__sub__wrapper"> <ul class="navigation__sub__columns"> <li data-gt-secondary="Research areas"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading">Research areas</h2> <ul> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/" > Explore all research areas </a> </li> </ul> </div> <div class="navigation__sub__columns__mobile"> <button class="glue-header__link js-sub-nav-target" data-panel="nested" role="menuitem" aria-haspopup="true"> Research areas <span class="icon icon--caret"></span> </button> <div class="navigation__nested-sub js-sub-nav-parent"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile" role="menuitem" aria-haspopup="true"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Research areas <span class="sr-text">menu</span> </button> <hr/> </div> <ul> <li role="menuitem"> <a href="https://research.google/research-areas/" class="navigation__sub__columns__mobile__link" > Explore all research areas <span> </span> </a> </li> </ul> </div> </div> </li> <li data-gt-secondary="Foundational ML & Algorithms"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading">Foundational ML & Algorithms</h2> <ul> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/algorithms-and-theory/" > Algorithms & Theory </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/data-management/" > Data Management </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/data-mining-and-modeling/" > Data Mining & Modeling </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/information-retrieval-and-the-web/" > Information Retrieval & the Web </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/machine-intelligence/" > Machine Intelligence </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/machine-perception/" > Machine Perception </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/machine-translation/" > Machine Translation </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/natural-language-processing/" > Natural Language Processing </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/speech-processing/" > Speech Processing </a> </li> </ul> </div> <div class="navigation__sub__columns__mobile"> <button class="glue-header__link js-sub-nav-target" data-panel="nested" role="menuitem" aria-haspopup="true"> Foundational ML & Algorithms <span class="icon icon--caret"></span> </button> <div class="navigation__nested-sub js-sub-nav-parent"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile" role="menuitem" aria-haspopup="true"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Foundational ML & Algorithms <span class="sr-text">menu</span> </button> <hr/> </div> <ul> <li role="menuitem"> <a href="https://research.google/research-areas/algorithms-and-theory/" class="navigation__sub__columns__mobile__link" > Algorithms & Theory <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/data-management/" class="navigation__sub__columns__mobile__link" > Data Management <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/data-mining-and-modeling/" class="navigation__sub__columns__mobile__link" > Data Mining & Modeling <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/information-retrieval-and-the-web/" class="navigation__sub__columns__mobile__link" > Information Retrieval & the Web <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/machine-intelligence/" class="navigation__sub__columns__mobile__link" > Machine Intelligence <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/machine-perception/" class="navigation__sub__columns__mobile__link" > Machine Perception <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/machine-translation/" class="navigation__sub__columns__mobile__link" > Machine Translation <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/natural-language-processing/" class="navigation__sub__columns__mobile__link" > Natural Language Processing <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/speech-processing/" class="navigation__sub__columns__mobile__link" > Speech Processing <span> </span> </a> </li> </ul> </div> </div> </li> <li data-gt-secondary="Computing Systems & Quantum AI"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading">Computing Systems & Quantum AI</h2> <ul> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/distributed-systems-and-parallel-computing/" > Distributed Systems & Parallel Computing </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/hardware-and-architecture/" > Hardware & Architecture </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/mobile-systems/" > Mobile Systems </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/networking/" > Networking </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/quantum-computing/" > Quantum Computing </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/robotics/" > Robotics </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/security-privacy-and-abuse-prevention/" > Security, Privacy, & Abuse Prevention </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/software-engineering/" > Software Engineering </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/software-systems/" > Software Systems </a> </li> </ul> </div> <div class="navigation__sub__columns__mobile"> <button class="glue-header__link js-sub-nav-target" data-panel="nested" role="menuitem" aria-haspopup="true"> Computing Systems & Quantum AI <span class="icon icon--caret"></span> </button> <div class="navigation__nested-sub js-sub-nav-parent"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile" role="menuitem" aria-haspopup="true"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Computing Systems & Quantum AI <span class="sr-text">menu</span> </button> <hr/> </div> <ul> <li role="menuitem"> <a href="https://research.google/research-areas/distributed-systems-and-parallel-computing/" class="navigation__sub__columns__mobile__link" > Distributed Systems & Parallel Computing <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/hardware-and-architecture/" class="navigation__sub__columns__mobile__link" > Hardware & Architecture <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/mobile-systems/" class="navigation__sub__columns__mobile__link" > Mobile Systems <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/networking/" class="navigation__sub__columns__mobile__link" > Networking <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/quantum-computing/" class="navigation__sub__columns__mobile__link" > Quantum Computing <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/robotics/" class="navigation__sub__columns__mobile__link" > Robotics <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/security-privacy-and-abuse-prevention/" class="navigation__sub__columns__mobile__link" > Security, Privacy, & Abuse Prevention <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/software-engineering/" class="navigation__sub__columns__mobile__link" > Software Engineering <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/software-systems/" class="navigation__sub__columns__mobile__link" > Software Systems <span> </span> </a> </li> </ul> </div> </div> </li> <li data-gt-secondary="Science, AI & Society"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading">Science, AI & Society</h2> <ul> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/climate-and-sustainability/" > Climate & Sustainability </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/economics-and-electronic-commerce/" > Economics & Electronic Commerce </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/education-innovation/" > Education Innovation </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/general-science/" > General Science </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/health-bioscience/" > Health & Bioscience </a> </li> <li> <a class="navigation__sub__columns__list-link caption js-drawer-link" href="https://research.google/research-areas/human-computer-interaction-and-visualization/" > Human-Computer Interaction and Visualization </a> </li> </ul> </div> <div class="navigation__sub__columns__mobile"> <button class="glue-header__link js-sub-nav-target" data-panel="nested" role="menuitem" aria-haspopup="true"> Science, AI & Society <span class="icon icon--caret"></span> </button> <div class="navigation__nested-sub js-sub-nav-parent"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile" role="menuitem" aria-haspopup="true"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Science, AI & Society <span class="sr-text">menu</span> </button> <hr/> </div> <ul> <li role="menuitem"> <a href="https://research.google/research-areas/climate-and-sustainability/" class="navigation__sub__columns__mobile__link" > Climate & Sustainability <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/economics-and-electronic-commerce/" class="navigation__sub__columns__mobile__link" > Economics & Electronic Commerce <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/education-innovation/" class="navigation__sub__columns__mobile__link" > Education Innovation <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/general-science/" class="navigation__sub__columns__mobile__link" > General Science <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/health-bioscience/" class="navigation__sub__columns__mobile__link" > Health & Bioscience <span> </span> </a> </li> <li role="menuitem"> <a href="https://research.google/research-areas/human-computer-interaction-and-visualization/" class="navigation__sub__columns__mobile__link" > Human-Computer Interaction and Visualization <span> </span> </a> </li> </ul> </div> </div> </li> </ul> </div> </div></div> </div> </div> </li> <li class="glue-header__item js-sub-nav-parent --parent" data-gt-primary="Our work" > <button class="glue-header__link js-sub-nav-target" aria-haspopup="true" aria-expanded="false" > <span class=""> Our work <span class="icon icon--caret"></span> </span> </button> <div class="navigation__sub js-sub-nav" role="menu"> <div class="navigation__sub__container"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Our work <span class="sr-text">menu</span> </button> <hr/> </div> <div class="block-nav_drawer_columns_content"> <div class="navigation__sub--content" data-gt-secondary=""> <div class="navigation__sub__wrapper"> <ul class="navigation__sub__columns"> <li data-gt-secondary="Projects"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Projects </h2> <p class="navigation__sub__columns__description caption">We regularly open-source projects with the broader research community and apply our developments to Google products.</p> <a href="https://research.google/resources/our-projects/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Projects</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/resources/our-projects/" > Projects </a> </div> </li> <li data-gt-secondary="Publications"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Publications </h2> <p class="navigation__sub__columns__description caption">Publishing our work allows us to share ideas and work collaboratively to advance the field of computer science.</p> <a href="https://research.google/pubs/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Publications</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/pubs/" > Publications </a> </div> </li> <li data-gt-secondary="Resources"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Resources </h2> <p class="navigation__sub__columns__description caption">We make products, tools, and datasets available to everyone with the goal of building a more collaborative ecosystem.</p> <a href="https://research.google/resources/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Resources</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/resources/" > Resources </a> </div> </li> </ul> </div> </div> </div> </div> </div> </li> <li class="glue-header__item js-sub-nav-parent --parent" data-gt-primary="Programs & events" > <button class="glue-header__link js-sub-nav-target" aria-haspopup="true" aria-expanded="false" > <span class=""> Programs & events <span class="icon icon--caret"></span> </span> </button> <div class="navigation__sub js-sub-nav" role="menu"> <div class="navigation__sub__container"> <div class="navigation__sub__mobile-heading"> <button class="glue-header__link js-sub-nav-close-mobile"> <span class="sr-text">Back to</span> <span class="icon icon--caret"></span> Programs & events <span class="sr-text">menu</span> </button> <hr/> </div> <div class="block-nav_drawer_columns_content"> <div class="navigation__sub--content" data-gt-secondary="Shaping the future, together."> <div class="navigation__sub__wrapper"> <div class="navigation__sub__heading"> <h2 class="headline-3">Shaping the future, together.</h2> <a href="https://research.google/programs-and-events/" class="js-drawer-link" > Collaborate with us </a> </div> <ul class="navigation__sub__columns"> <li data-gt-secondary="Student programs"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Student programs </h2> <p class="navigation__sub__columns__description caption">Supporting the next generation of researchers through a wide range of programming.</p> <a href="https://research.google/programs-and-events/student-engagement/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Student programs</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/programs-and-events/student-engagement/" > Student programs </a> </div> </li> <li data-gt-secondary="Faculty programs"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Faculty programs </h2> <p class="navigation__sub__columns__description caption">Participating in the academic research community through meaningful engagement with university faculty.</p> <a href="https://research.google/programs-and-events/faculty-engagement/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Faculty programs</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/programs-and-events/faculty-engagement/" > Faculty programs </a> </div> </li> <li data-gt-secondary="Conferences & events"> <div class="navigation__sub__columns__desktop"> <h2 class="headline-6 navigation__sub__columns__heading"> Conferences & events </h2> <p class="navigation__sub__columns__description caption">Connecting with the broader research community through events is essential for creating progress in every aspect of our work.</p> <a href="https://research.google/conferences-and-events/" class="glue-inline-link js-drawer-link" > <span class="sr-text">Learn more about our Conferences & events</span> <span aria-hidden="true">Learn more</span> </a> </div> <div class="navigation__sub__columns__mobile"> <a class="glue-header__link" href="https://research.google/conferences-and-events/" > Conferences & events </a> </div> </li> </ul> <div class="navigation__sub__cta"> <a class="glue-button glue-button--high-emphasis js-drawer-link" href="https://research.google/programs-and-events/" target="_blank" rel="noreferrer noopener" > Collaborate with us </a> </div> </div> </div> </div> </div> </div> </li> <li class="glue-header__item " data-gt-primary="Careers" > <a class="glue-header__link " href="https://research.google/careers/" > <span class=""> Careers </span> </a> </li> <li class="glue-header__item " data-gt-primary="Blog" > <a class="glue-header__link " href="https://research.google/blog/" > <span class=""> Blog </span> </a> </li> </ul> </nav> </div> <!-- search (hide on search page) --> <div class="glue-header__search js-header-search"> <div class="glue-header__search__input"> <div class="search-input " data-type="header"> <input type="search" class="caption --empty-search js-search-bar js-gt-search-input" placeholder="Search"> <button class="search-input__button --search js-gt-search-btn"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--18px "> <use href="/gr/static/assets/icons/glue-icons.svg#search"></use> </svg> </button> <button class="search-input__button --clear"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--18px "> <use href="/gr/static/assets/icons/glue-icons.svg#close"></use> </svg> </button> </div> </div> <button class="glue-header__search__btn js-header-search-btn"> <svg role="presentation" aria-hidden="true" aria-hidden="true" class="glue-icon glue-icon--24px search"> <use href="/gr/static/assets/icons/glue-icons.svg#search"></use> </svg> <svg role="presentation" aria-hidden="true" aria-hidden="true" class="glue-icon glue-icon--24px close"> <use href="/gr/static/assets/icons/glue-icons.svg#close"></use> </svg> <span class="sr-text js-header-search-sr-text">Search</span> </button> </div> </div> </div> <div class="glue-header__drawer-backdrop"> <div class="glue-header__mobile_close"> <button class="glue-header__drawer-toggle-btn js-mobile-nav-close" aria-label="Close the navigation drawer"> <svg class="glue-icon glue-icon--24px" role="presentation" aria-hidden="true"> <use href="/gr/static/assets/icons/glue-icons.svg#close"></use> </svg> </button> </div> </div> </header> <main id="page-content"> <div class="blog-detail-page --legacy " > <section class="basic-hero bhoig --theme-dark --large-image" data-gt-id="basic_hero" data-gt-component-name=""> <div class="glue-page"> <div class="glue-grid"> <div class="bhoig__image-wrapper glue-grid__col--span-4 glue-grid__col--span-5-md glue-grid__col--span-4-lg"> <div class="bhoig__image-bg" style=""> <picture> <img src="https://storage.googleapis.com/gweb-research2023-media/original_images/bd1f9a0da60ef93b224d8e4a484f4860-GoogleResearch.png" alt="" class=""/> </picture> </div> </div> <div class="bhoig__breadcrumb-wrapper glue-grid__col--span-10 glue-grid__col--span-9-md glue-grid__col--span-10-lg"> <nav class="glue-breadcrumbs" aria-label="Breadcrumbs"> <ol class="glue-breadcrumbs__list"> <li class="glue-breadcrumbs__item"> <a class="glue-breadcrumbs__link attribution" href="/">Home</a> <svg role="presentation" aria-hidden="true" class="glue-icon "> <use href="/gr/static/assets/icons/glue-icons.svg#chevron-right"></use> </svg> </li> <li class="glue-breadcrumbs__item"> <a class="glue-breadcrumbs__link attribution" href="/blog/">Blog</a> <svg role="presentation" aria-hidden="true" class="glue-icon "> <use href="/gr/static/assets/icons/glue-icons.svg#chevron-right"></use> </svg> </li> </ol> </nav> </div> <h1 class="headline-1 bhoig__headline glue-grid__col--span-8 glue-grid__col--span-7-md glue-grid__col--span-8-lg">Supporting benchmarks for AI safety with MLCommons</h1> <div class="basic-hero__description bhoig__description glue-grid__col--span-8 glue-grid__col--span-7-md glue-grid__col--span-8-lg"> <div class="basic-hero--blog-detail__description"><p>October 26, 2023</p><span class="dot-separator"></span><p>Posted by Anoop Sinha, Technology and Society, and Marian Croak, Google Research, Responsible AI and Human Centered Technology team</p></div> </div> <div class="bhoig__cta glue-grid__col--span-8 glue-grid__col--span-7-md glue-grid__col--span-8-lg"> </div> </div> </div> </section> <div class="glue-page"> <div class="glue-grid blog-detail-page__grid"> <div class="glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-12-md glue-grid__col--span-9-lg"> <div class="quicklinks-wrapper--mobile"> <div class="block-quick_links"> <section class="quicklinks"> <h2 class="eyebrow">Quick links</h2> <ul class="quicklinks__list"> <li class="quicklinks__item quicklinks__item--share js-quicklinks-share"> <button class="quicklinks__share-button js-quicklinks-share__button" aria-expanded="false" aria-controls="js-quicklinks-share__list"> <span class="icon icon--share"></span> <span class="quicklinks__item__text">Share</span> </button> <section class="glue-social glue-social--monochrome quicklinks__share-list js-quicklinks-share__list glue-elevation-level-1 js-gt-share-wrapper"> <div class="glue-social__group"> <ul class="glue-social__list" role="list"> <li class="glue-social__item"> <a class="glue-social__link" href="https://twitter.com/intent/tweet?text=https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" title="Share on Twitter" target="_blank" rel="noopener" data-gt-method="x"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--24px"> <use href="/gr/static/assets/icons/twitter-x.svg#twitter-x"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://www.facebook.com/sharer/sharer.php?u=https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" title="Share on Facebook" target="_blank" rel="noopener" data-gt-method="facebook"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-facebook glue-icon--24px"> <use href="/gr/static/assets/icons/facebook.svg#facebook"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://www.linkedin.com/shareArticle?url=https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/&mini=true" title="Share on LinkedIn" target="_blank" rel="noopener" data-gt-method="linkedin"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-linkedin glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#post-linkedin"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="mailto:name@example.com?subject=Check%20out%20this%20site&body=Check%20out%20https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" title="Send via Email" data-gt-method="email"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-sharemail glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#email"></use> </svg> </a> </li> <li class="glue-social__item"> <div class="glue-social__popover"> <div class="glue-social__icon-trigger" aria-label="Get shareable link" title="Get shareable link" id="share-static-popover-trigger"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-sharelink glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#link"></use> </svg> </div> <div class="glue-social__dialog" id="share-popover-dialog"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-sharelink glue-icon--24px"> <use href="/public/icons/glue-icons.svg#link"></use> </svg> <div class="glue-social__copy" glue-copy-success="Copied to clipboard" glue-copy-fail="Press Ctrl+C or ⌘+C to copy"> <input class="glue-social__copy-input" readonly="" type="text" value="https://research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" aria-label="URL"> <button class="glue-social__copy-btn" id="share-copy-btn" data-gt-method="link-copied">Copy link</button> </div> <div aria-label="Close" class="glue-social__close-btn"> × </div> </div> </div> </li> </ul> </div> </section> </li> </ul> </section> </div> </div> <div class="blog-detail-wrapper js-gt-blog-detail-wrapper" data-gt-publish-date="20231026"> <div class="rich-text --theme- --mode-" data-gt-id="rich_text" data-gt-component-name=""> <img src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg5URA9ivhxcgmfXwa0668O0HslgYQ_p_x9JJbg6nDgryyNc2QImQernPyzkLPcj1esCMOQTUnEsldIlb21E0DWPDIiE2m76qSruF0jA6jfl1sNl6mBW8JUPSWzOfE7IahHRtviFpxUTPcEZoADXHNMy3gZ2hg369y5QxhY01QRVj5kwJx4uwRKzdcBFp9v/s1600/GoogleResearch.png" style="display: none;" /> <p> Standard benchmarks are agreed upon ways of measuring important product qualities, and they exist in many fields. Some standard benchmarks measure safety: for example, when a car manufacturer touts a “five-star overall safety rating,” they’re citing a benchmark. Standard benchmarks already exist in machine learning (ML) and AI technologies: for instance, the <a href="https://mlcommons.org/en/" target="_blank" rel="noopener noreferrer">MLCommons</a> Association operates the <a href="https://mlcommons.org/en/news/mlperf-inference-storage-q323/" target="_blank" rel="noopener noreferrer">MLPerf</a> benchmarks that measure the speed of cutting edge AI hardware such as Google’s TPUs. However, though there has been significant work done on <a href="https://blog.google/technology/ai/our-responsible-approach-to-building-guardrails-for-generative-ai/" target="_blank" rel="noopener noreferrer">AI safety</a>, there are as yet no similar standard benchmarks for AI safety. </p> <a name='more'></a> <p> We are excited to support a new effort by the non-profit MLCommons Association to develop standard AI safety benchmarks. Developing benchmarks that are effective and trusted is going to require advancing AI safety testing technology and incorporating a broad range of perspectives. The MLCommons effort aims to bring together expert researchers across academia and industry to develop standard benchmarks for measuring the safety of AI systems into scores that everyone can understand. We encourage the whole community, from AI researchers to policy experts, to <a href="https://mlcommons.org/ai-safety" target="_blank" rel="noopener noreferrer">join us</a> in contributing to the effort. </p> <div style="line-height:40%;"> <br> </div> <h2>Why AI safety benchmarks?</h2> <p> Like most advanced technologies, AI has the potential for tremendous benefits but could also lead to negative outcomes without appropriate care. For example, AI technology can boost human productivity in a wide range of activities (e.g., <a href="https://blog.google/technology/health/how-ai-can-improve-health-for-everyone-everywhere/" target="_blank" rel="noopener noreferrer">improve health diagnostics</a> and research into diseases, analyze <a href="https://blog.google/outreach-initiatives/sustainability/google-transportation-energy-emissions-reduction/" target="_blank" rel="noopener noreferrer">energy usage</a>, and more). However, without sufficient precautions, AI could also be used to support harmful or malicious activities and respond in biased or offensive ways. </p> <p> By providing standard measures of safety across categories such as harmful use, out-of-scope responses, AI-control risks, etc., standard AI safety benchmarks could help society reap the benefits of AI while ensuring that sufficient precautions are being taken to mitigate these risks. Initially, nascent safety benchmarks could help drive AI safety research and inform responsible AI development. With time and maturity, they could help inform users and purchasers of AI systems. Eventually, they could be a valuable tool for policy makers. </p> <p> In computer hardware, benchmarks (e.g., <a href="https://en.wikipedia.org/wiki/Standard_Performance_Evaluation_Corporation" target="_blank" rel="noopener noreferrer">SPEC</a>, <a href="https://en.wikipedia.org/wiki/Transaction_Processing_Performance_Council" target="_blank" rel="noopener noreferrer">TPC</a>) have shown an amazing ability to align research, engineering, and even marketing across an entire industry in pursuit of progress, and we believe standard AI safety benchmarks could help do the same in this vital area. </p> <div style="line-height:40%;"> <br> </div> <h2>What are standard AI safety benchmarks?</h2> <p> Academic and corporate research efforts have experimented with a range of AI safety tests (e.g., <a href="https://arxiv.org/abs/2009.11462" target="_blank" rel="noopener noreferrer">RealToxicityPrompts</a>, <a href="https://crfm.stanford.edu/2022/11/17/helm.html" target="_blank" rel="noopener noreferrer">Stanford HELM</a> fairness, bias, toxicity measurements, and <a href="https://blog.google/technology/ai/our-responsible-approach-to-building-guardrails-for-generative-ai/" target="_blank" rel="noopener noreferrer">Google’s guardrails for generative AI</a>). However, most of these tests focus on providing a prompt to an AI system and algorithmically scoring the output, which is a useful start but limited to the scope of the test prompts. Further, they usually use open datasets for the prompts and responses, which may already have been (often inadvertently) incorporated into training data. </p> <p> MLCommons proposes a multi-stakeholder process for selecting tests and grouping them into subsets to measure safety for particular AI use-cases, and translating the highly technical results of those tests into scores that everyone can understand. MLCommons is proposing to create a platform that brings these existing tests together in one place and encourages the creation of more rigorous tests that move the state of the art forward. Users will be able to access these tests both through online testing where they can generate and review scores and offline testing with an engine for private testing. </p> <div style="line-height:40%;"> <br> </div> <h2>AI safety benchmarks should be a collective effort</h2> <p> Responsible AI developers use a diverse range of safety measures, including automatic testing, manual testing, red teaming (in which human testers attempt to produce adversarial outcomes), software-imposed restrictions, data and model best-practices, and auditing. However, determining that sufficient precautions have been taken can be challenging, especially as the community of companies providing AI systems grows and diversifies. Standard AI benchmarks could provide a powerful tool for helping the community grow responsibly, both by helping vendors and users measure AI safety and by encouraging an ecosystem of resources and specialist providers focused on improving AI safety. </p> <p> At the same time, development of mature AI safety benchmarks that are both effective and trusted is not possible without the involvement of the community. This effort will need researchers and engineers to come together and provide innovative yet practical improvements to safety testing technology that make testing both more rigorous and more efficient. Similarly, companies will need to come together and provide test data, engineering support, and financial support. Some aspects of AI safety can be subjective, and building trusted benchmarks supported by a broad consensus will require incorporating multiple perspectives, including those of public advocates, policy makers, academics, engineers, data workers, business leaders, and entrepreneurs. </p> <div style="line-height:40%;"> <br> </div> <h2>Google’s support for MLCommons</h2> <p> Grounded in our <a href="https://ai.google/responsibility/principles/" target="_blank" rel="noopener noreferrer">AI Principles</a> that were <a href="https://blog.google/technology/ai/ai-principles/" target="_blank" rel="noopener noreferrer">announced</a> in 2018, Google is committed to specific practices for the safe, secure, and trustworthy development and use of AI (see our <a href="https://ai.google/static/documents/ai-principles-2019-progress-update.pdf" target="_blank" rel="noopener noreferrer">2019</a>, <a href="https://ai.google/static/documents/ai-principles-2020-progress-update.pdf" target="_blank" rel="noopener noreferrer">2020</a>, <a href="https://ai.google/static/documents/ai-principles-2021-progress-update.pdf" target="_blank" rel="noopener noreferrer">2021</a>, <a href="https://ai.google/static/documents/ai-principles-2022-progress-update.pdf" target="_blank" rel="noopener noreferrer">2022</a> updates). We’ve also made significant <a href="https://static.googleusercontent.com/media/publicpolicy.google/en//resources/whcommitments.pdf" target="_blank" rel="noopener noreferrer">progress</a> on key commitments, which will help ensure AI is developed boldly and responsibly, for the benefit of everyone. </p> <p> Google is supporting the MLCommons Association's efforts to develop AI safety benchmarks in a number of ways. </p> <ol> <li><em>Testing platform</em>: We are joining with other companies in providing funding to support the development of a testing platform. <li><em>Technical expertise and resources</em>: We are providing technical expertise and resources, such as the <a href="https://skintone.google/mste-dataset" target="_blank" rel="noopener noreferrer">Monk Skin Tone Examples Dataset</a>, to help ensure that the benchmarks are well-designed and effective. <li><em>Datasets</em>: We are contributing an internal dataset for multilingual representational bias, as well as already externalized tests for stereotyping harms, such as <a href="https://github.com/google-research-datasets/seegull/tree/main" target="_blank" rel="noopener noreferrer">SeeGULL</a> and <a href="https://github.com/google-research-datasets/SPICE/tree/main" target="_blank" rel="noopener noreferrer">SPICE</a>. Moreover, we are sharing our datasets that focus on collecting human annotations responsibly and inclusively, like <a href="https://arxiv.org/abs/2306.11247" target="_blank" rel="noopener noreferrer">DICES</a> and <a href="https://www.kaggle.com/datasets/google/jigsaw-specialized-rater-pools-dataset" target="_blank" rel="noopener noreferrer">SRP</a>. </li> </ol> <div style="line-height:40%;"> <br> </div> <h2>Future direction</h2> <p> We believe that these benchmarks will be very useful for advancing research in AI safety and ensuring that AI systems are developed and deployed in a responsible manner. AI safety is <a href="https://blog.google/technology/ai/a-shared-agenda-for-responsible-ai-progress/" target="_blank" rel="noopener noreferrer">a collective-action problem</a>. Groups like the <a href="https://blog.google/outreach-initiatives/public-policy/google-microsoft-openai-anthropic-frontier-model-forum/" target="_blank" rel="noopener noreferrer">Frontier Model Forum</a> and <a href="https://partnershiponai.org/" target="_blank" rel="noopener noreferrer">Partnership on AI</a> are also leading important standardization initiatives. We’re pleased to have been part of these groups and MLCommons since their beginning. We look forward to additional collective efforts to promote the responsible development of new generative AI tools. </p> <div style="line-height:40%;"> <br> </div> <h2>Acknowledgements</h2> <p> <em>Many thanks to the Google team that contributed to this work: Peter Mattson, Lora Aroyo, Chris Welty, Kathy Meier-Hellstern, Parker Barnes, Tulsee Doshi, Manvinder Singh, Brian Goldman, Nitesh Goyal, Alice Friend, Nicole Delange, Kerry Barker, Madeleine Elish, Shruti Sheth, Dawn Bloxwich, William Isaac, Christina Butterfield.</em> </p> </div> </div> <section aria-label="List of footnotes" data-gt-id="footnotes" data-gt-component-name="Footnotes"> <ol class="js-footnotes footnotes"> </ol> </section> <section class="blog-labels" data-gt-id="blog_labels" data-gt-component-name="Blog Labels"> <ul class="blog-labels__list"> <span class="caption">Labels:</span> <li class="caption"> <a class="caption" href="/blog/label/machine-intelligence">Machine Intelligence</a> <div class="blog-labels__spacer"></div> </li> <li class="caption"> <a class="caption" href="/blog/label/responsible-ai">Responsible AI</a> </li> </ul> </section> </div> <div class="glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-12-md glue-grid__col--span-3-lg"> <div class="quicklinks-wrapper--desktop quicklinks-wrapper--sticky"> <div class="block-quick_links"> <section class="quicklinks"> <h2 class="eyebrow">Quick links</h2> <ul class="quicklinks__list"> <li class="quicklinks__item quicklinks__item--share js-quicklinks-share"> <button class="quicklinks__share-button js-quicklinks-share__button" aria-expanded="false" aria-controls="js-quicklinks-share__list"> <span class="icon icon--share"></span> <span class="quicklinks__item__text">Share</span> </button> <section class="glue-social glue-social--monochrome quicklinks__share-list js-quicklinks-share__list glue-elevation-level-1 js-gt-share-wrapper"> <div class="glue-social__group"> <ul class="glue-social__list" role="list"> <li class="glue-social__item"> <a class="glue-social__link" href="https://twitter.com/intent/tweet?text=https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" title="Share on Twitter" target="_blank" rel="noopener" data-gt-method="x"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--24px"> <use href="/gr/static/assets/icons/twitter-x.svg#twitter-x"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://www.facebook.com/sharer/sharer.php?u=https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" title="Share on Facebook" target="_blank" rel="noopener" data-gt-method="facebook"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-facebook glue-icon--24px"> <use href="/gr/static/assets/icons/facebook.svg#facebook"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://www.linkedin.com/shareArticle?url=https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/&mini=true" title="Share on LinkedIn" target="_blank" rel="noopener" data-gt-method="linkedin"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-linkedin glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#post-linkedin"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="mailto:name@example.com?subject=Check%20out%20this%20site&body=Check%20out%20https%3A//research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" title="Send via Email" data-gt-method="email"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-sharemail glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#email"></use> </svg> </a> </li> <li class="glue-social__item"> <div class="glue-social__popover"> <div class="glue-social__icon-trigger" aria-label="Get shareable link" title="Get shareable link" id="share-static-popover-trigger"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-sharelink glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#link"></use> </svg> </div> <div class="glue-social__dialog" id="share-popover-dialog"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--color-sharelink glue-icon--24px"> <use href="/public/icons/glue-icons.svg#link"></use> </svg> <div class="glue-social__copy" glue-copy-success="Copied to clipboard" glue-copy-fail="Press Ctrl+C or ⌘+C to copy"> <input class="glue-social__copy-input" readonly="" type="text" value="https://research.google/blog/supporting-benchmarks-for-ai-safety-with-mlcommons/" aria-label="URL"> <button class="glue-social__copy-btn" id="share-copy-btn" data-gt-method="link-copied">Copy link</button> </div> <div aria-label="Close" class="glue-social__close-btn"> × </div> </div> </div> </li> </ul> </div> </section> </li> </ul> </section> </div> </div> </div> </div> </div> <section class="related-posts offset-two-up --theme-dark" data-gt-id="related_blog_posts" data-gt-component-name="Related Blog Posts"> <div class="glue-page glue-grid"> <div class="offset-two-up__left-col glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-12-md glue-grid__col--span-3-lg"> <h3 class="offset-two-up__headline headline-3">Other posts of interest</h3> </div> <div class="glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-12-md glue-grid__col--span-9-lg"> <ul class="card-stack--basic nested-glue-grid-override"> <li class="glue-grid__col glue-grid__col--span-4-md glue-grid__col--span-4-sm"> <a class="glue-card not-glue " href="/blog/building-ai-for-the-pluralistic-society/" aria-label="" > <div class="glue-card__inner"> <div class="related-posts__image"> <img src="https://storage.googleapis.com/gweb-research2023-media/original_images/PluralisticAI-1-Objectives.png" alt="" /> </div> <div class="glue-card__content --no-media"> <p class="glue-label glue-spacer-1-bottom">February 13, 2025</p> <span class="headline-5 js-gt-item-id"> Building AI for the pluralistic society </span> </div> <ul class="glue-card__link-list"> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Generative AI <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Natural Language Processing <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Responsible AI </span> </li> </ul> </div> </a> </li> <li class="glue-grid__col glue-grid__col--span-4-md glue-grid__col--span-4-sm"> <a class="glue-card not-glue " href="/blog/chain-of-agents-large-language-models-collaborating-on-long-context-tasks/" aria-label="" > <div class="glue-card__inner"> <div class="related-posts__image"> <img src="https://storage.googleapis.com/gweb-research2023-media/original_images/CoA-5-HotpotQA.png" alt="" /> </div> <div class="glue-card__content --no-media"> <p class="glue-label glue-spacer-1-bottom">January 23, 2025</p> <span class="headline-5 js-gt-item-id"> Chain of Agents: Large language models collaborating on long-context tasks </span> </div> <ul class="glue-card__link-list"> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Generative AI <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Machine Intelligence <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Natural Language Processing </span> </li> </ul> </div> </a> </li> <li class="glue-grid__col glue-grid__col--span-4-md glue-grid__col--span-4-sm"> <a class="glue-card not-glue " href="/blog/parfait-enabling-private-ai-with-research-tools/" aria-label="" > <div class="glue-card__inner"> <div class="related-posts__image"> <img src="https://storage.googleapis.com/gweb-research2023-media/original_images/Parfait1-PreviewHero.png" alt="" /> </div> <div class="glue-card__content --no-media"> <p class="glue-label glue-spacer-1-bottom">January 22, 2025</p> <span class="headline-5 js-gt-item-id"> Parfait: Enabling private AI with research tools </span> </div> <ul class="glue-card__link-list"> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Distributed Systems & Parallel Computing <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Generative AI <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Responsible AI <span class="glue-card__link-list__spacer">·</span> </span> </li> <li class="glue-card__link-list__item"> <span class="not-glue caption"> Security, Privacy and Abuse Prevention </span> </li> </ul> </div> </a> </li> </ul> </div> </div> </section> </div> </main> <footer class="glue-footer"> <div class="glue-page"> <section class="glue-social"> <div class="glue-social__group glue-social--monochrome"> <p class="glue-social__title glue-social__title--inline"> Follow us </p> <nav class="js-gt-follow-us-wrapper" aria-label="Social media links"> <ul class="glue-social__list" role="list"> <li class="glue-social__item"> <a class="glue-social__link" href="https://twitter.com/GoogleAI" title="Follow us on x" target="_blank" rel="noopener" data-gt-method="x"" > <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--24px"> <use href="/gr/static/assets/icons/twitter-x.svg#twitter-x"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://www.linkedin.com/showcase/googleresearch/" title="Follow us on linkedin" target="_blank" rel="noopener" data-gt-method="linkedin"" > <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#post-linkedin"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://www.youtube.com/c/GoogleResearch" title="Follow us on youtube" target="_blank" rel="noopener" data-gt-method="youtube"" > <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--24px"> <use href="/gr/static/assets/icons/glue-icons.svg#video-youtube"></use> </svg> </a> </li> <li class="glue-social__item"> <a class="glue-social__link" href="https://github.com/google-research" title="Follow us on github" target="_blank" rel="noopener" data-gt-method="github"" > <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--24px"> <use href="/gr/static/assets/icons/github.svg#github"></use> </svg> </a> </li> </ul> </nav> </div> </section> </div> <div class="glue-fullbleed"></div> <section class="glue-page"> <nav class="glue-footer__global" aria-label="Footer resource links"> <div class="glue-footer__logo"> <a href="https://www.google.com" title="Google" class="glue-footer__link"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-footer__logo-img"> <use href="/gr/static/assets/icons/glue-icons.svg#google-solid-logo"></use> </svg> </a> </div> <ul class="glue-footer__global-links glue-no-bullet js-gt-global-nav-wrapper" role="list"> <li class="glue-footer__global-links-list-item" data-gt-primary="About Google"> <a class="glue-footer__link" href="https://about.google/" target="_blank" rel="noopener"> About Google </a> </li> <li class="glue-footer__global-links-list-item" data-gt-primary="Google Products"> <a class="glue-footer__link" href="https://about.google/intl/en/products/" target="_blank" rel="noopener"> Google Products </a> </li> <li class="glue-footer__global-links-list-item" data-gt-primary="Privacy"> <a class="glue-footer__link" href="https://policies.google.com/privacy" target="_blank" rel="noopener"> Privacy </a> </li> <li class="glue-footer__global-links-list-item" data-gt-primary="Terms"> <a class="glue-footer__link" href="https://policies.google.com/terms" target="_blank" rel="noopener"> Terms </a> </li> </ul> <ul class="glue-footer__global-links glue-footer__global-links--extra glue-no-bullet" role="list"> <li class="glue-footer__global-links-list-item glue-footer__global-links-list-item--extra"> <a class="glue-footer__link" href="https://support.google.com/?hl=en"> <svg role="presentation" aria-hidden="true" aria-hidden="true" class="glue-icon glue-icon--24px glue-icon--footer-help"> <use href="/gr/static/assets/icons/glue-icons.svg#help"></use> </svg> Help </a> </li> <li class="glue-footer__global-links-list-item glue-footer__global-links-list-item--extra"> <button class="glue-footer__link google-feedback js-feedback-button" href="" data-product-id="5137383" > Submit feedback </button> </li> </ul> </nav> </section> </footer> <script src="https://www.gstatic.com/glue/v27_1/material-components-web.min.js"></script> <script src="https://www.youtube.com/player_api"></script> <script type="text/javascript" src="/gr/static/js/googleresearch.js?id=b70549917812130af912601ad763f13e"></script> <script type="text/javascript" src="https://support.google.com/inapp/api.js"></script> <script src="https://www.gstatic.com/glue/cookienotificationbar/cookienotificationbar.min.js" data-glue-cookie-notification-bar-category="2B"> </script> </body> </html>