CINXE.COM
The National Academies Press
<!doctype html> <html lang="en"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta http-equiv="cache-control" content="no-cache"/> <title>The National Academies Press</title> <script> // if (document.location.hostname.search(/nap\.edu$/) < 0) { // url = "http://www.na" + "p.edu" + document.location.pathname + document.location.search // document.location.replace(url); // } </script> <!--[if lt IE 9]> <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script> <script src="/includes/js/svg4everybody.ie8.min.js"></script> <![endif]--> <meta name="alexaVerifyID" content="02J2SepmgvpkhVXcfeE98cX3HKs" /> <link href="//nap.nationalacademies.org/stylesheets/nasem-nap.css" rel="stylesheet" type="text/css"> <link href="//nap.nationalacademies.org/stylesheets/topnav-lite-min.css" rel="stylesheet" type="text/css" /> <link href="//nap.nationalacademies.org/stylesheets/component.css" rel="stylesheet" type="text/css" /> <!--[if gte IE 5]> <style type="text/css"> #blanket {filter:alpha(opacity=65);}</style><![endif]--> <script src="/includes/js/jquery-3.5.1.min.js"></script> <script src="/includes/js/jquery.dotdotdot.min.js" type="text/javascript"></script> <!--<script src="https://www.nationalacademies.org//_cache_4662/_themesdelivery/nasuwTheme/assets/vendor.js"></script>--> <!-- <script src="/includes/js/jquery.min-3.1.0.js"></script> --> <!-- MC Goal Tracking --> <script type="text/javascript"> var $mcGoal = {'settings':{'uuid':'eaea39b6442dc4e0d08e6aa4a','dc':'us4'}}; (function() { var sp = document.createElement('script'); sp.type = 'text/javascript'; sp.async = true; sp.defer = true; sp.src = ('https:' == document.location.protocol ? 'https://s3.amazonaws.com/downloads.mailchimp.com' : 'http://downloads.mailchimp.com') + '/js/goal.min.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(sp, s); })(); </script> <!-- Data Layer --> <script> window.dataLayer = window.dataLayer || [];dataLayer.push({ 'pageTitle' : "Home", 'loginStatus' : "guest", 'form' : "subscribe" }); </script> <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-N9MJ23');</script> <!-- Global site tag (gtag.js) - Google AdWords: 1071160892 --> <script async src="https://www.googletagmanager.com/gtag/js?id=AW-1071160892"></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'AW-1071160892'); gtag('config', 'AW-1071160892'); </script> <script id="mcjs">!function(c,h,i,m,p){m=c.createElement(h),p=c.getElementsByTagName(h)[0],m.async=1,m.src=i,p.parentNode.insertBefore(m,p)}(document,"script","https://chimpstatic.com/mcjs-connected/js/users/eaea39b6442dc4e0d08e6aa4a/208cb1a4b431cae38b54bdb6f.js");</script> <link rel="stylesheet" href="/stylesheets/vendor/bootstrap-3.0.3/bootstrap.min.css"> <link rel="stylesheet" href="/stylesheets/naphome.css" /> <style> body { /*background-color: #FAF8F6; */ background-color: #f0ede5; } span.searchterm { color: #333; } .sibcol2 { background-color: #fff !important; } .chaphits { background: #fff; padding: 10px 60px 10px 10px; margin-bottom: 10px; } .chaphitshdr { background: #dedede; padding: 10px 60px 10px 10px; margin-bottom: 10px; } .chaphits h4, .chaphitshdr h4 { line-height: 1.3 !important; color: #666; } .btn-hits { color: #999; background-color: #efefef !important; border: none; border-radius: 0px !important; } .print a.buyit { width: 100%; background-color: #1d1646; color: white; padding: 5px; } .print a.buyit:hover { background: #7899F2; color: white; } a.buyit { display: inline-block; padding: 6px 12px 6px 35px; background: none; margin: 10px 0px; color: #fff; } /*.btn-nohits a { cursor: default; } */ .format { text-transform: capitalize !important; } .webprice { font-size: 16px; } a.dloadit { white-space: nowrap; } /*.btn-nohits a:hover { text-decoration: none; } */ .heading.row p.small { font-size: 90%; color: #999; padding: 5px 0px 5px 0px; margin: 0px; } .form-group.searchinside { width: 90%; } .searchbook { display: none; border-top: 1px solid #ccc; padding: 10px 0px 10px 0px; background: #ccc; margin-bottom: 20px; } .sib_chapname { padding-top: 5px; font-size: 110%; } .sib_chapname_nohits, .btn-nohits { display: none; } .sib_chapname_nohits .viewlinks { display: none; } div.fromblock { line-height: 18pt; color: #000; font-style: italic; font-weight: bold; } .searchagain { font-size: 12px; } .chaphit_desc { border-top: 1px solid #ccc; border-bottom: 1px solid #ccc; padding-left: 0px; } .chaphit_row { margin-bottom: 20px; display: block; } .chaphit_row_nodisplay { margin: 0px; display: none; } .hitlink { font-size: 85%; } .mask { height: 0px; opacity: 0; -moz-transition: all .5s ease-in-out; -webkit-transition: all .5s ease-in-out; position: absolute; bottom: 0px; left: 0; width: 100%; background-color: #4678aa; color: white; padding-top: 10px; } .btn-hits:hover .mask { opacity: .8; height: 100%; bottom: 0px; color: white; } .siccol2 {} .row.inchap_results { background: #fff; margin-bottom: 10px; padding-top: 10px; padding-bottom: 10px; } /*Styles to display page location*/ .termincontext { color: #F67F0B; font-family: 'archivo_narrowbold', helvetica, arial; } .sic_paragraph { color: #666; } #openbook_buying_info p { font-size: 12px; } </style> <style>pre { background-color: transparent !important; }</style> </head> <body> <!-- Google Tag Manager (nos cript) --> <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-N9MJ23" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <!-- End Google Tag Manager (noscript) --> <a name="top" id="top"></a> <a href="#content" class="sr-only sr-only-focusable">Skip to main content</a> <style> body { overflow-x: hidden; } #lowerMenu { background: #fff } .global-nav--local .mega-menu { display: block; -webkit-transform: translateY(0em); transform: translateY(0em); } .global-nav--local.search-is-hidden .mega-menu { display: none; -webkit-transform: translateY(0em); transform: translateY(0em); } .global-nav .container:not(.mega-menu__header) { padding: 0rem; padding-top: 0rem; padding-right: 0rem; padding-top: 0.5rem; padding-right: 1rem; } .global-nav__controls { top: .3rem; padding-right: 20px; } .global-nav--local .global-nav__links { -webkit-transform: translateY(0); transform: translateY(0); max-width: 75%; float: right; background: #fff; } .global-nav__links ul { display: -webkit-flex; display: flex; -webkit-justify-content: flex-end; justify-content: flex-end; margin-top: .5rem; } .global-nav--local.menu-is-hidden .global-nav__links { opacity: 0; -webkit-transform: translateY(0.5em); transform: translateY(0.5em); visibility: hidden; } #header-large { max-width: 1180px; margin: 0; padding: 0 0; -webkit-box-shadow: none; -moz-box-shadow: none; box-shadow: none; flex-wrap: nowrap; position: absolute; top: 113px; text-align: right; width: 100%; right: 0; } #header-large .user-links { white-space: nowrap; display: inline-flex; margin: 0; padding: 0; background: #fff; z-index: 0; list-style: none; min-width:500px; } .user-links li { margin-right: 7%; } .user-links li a { font-size: 10px; text-align: left; text-transform: uppercase; color: #000; } #header-large a.site-links { margin-right: 15px; color: #000; padding-left: 0px; padding-right: 5px; font-size: 12px; text-transform: uppercase; } .site-links:hover { text-decoration: underline; } nav#sitelinks-wide { position: absolute; top: 16px; } #sitelinks-wide ul { display: flex; justify-content: space-between; margin: 0; padding: 0; } #sitelinks-wide ul li { #sitelinks-mobile { display: none; } white-space: nowrap; } #sitelinks-mobile { display: none; } .user-links li a:hover { text-decoration: underline; } .dropdown-menu { position: absolute; top: 100%; left: 10px; z-index: 9999999; display: none; float: left; min-width: 160px; padding: 5px 0; margin: 2px 0 0; list-style: none; font-size: 14px; background-color: #ffffff; border: 1px solid #cccccc; border: 1px solid rgba(0, 0, 0, 0.15); border-radius: 4px; -webkit-box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175); box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175); background-clip: padding-box; } #dropdownMenuButton { border: 1px solid #ccc; font-size: 14px; font-weight: 700; } .dropdown #dropdownMenuButton:after { content: '\02795'; /* Unicode character for "plus" sign (+) */ font-size: 13px; color: #777; float: right; margin-left: 5px; } .dropdown.open #dropdownMenuButton:after { content: "\2796"; /* Unicode character for "minus" sign (-) */ } li.naphelp { display: block;} @media (max-width: 930px) { #header-large { right: 80px; } #header-large .user-links { min-width:100px; } nav#sitelinks-wide { display: none; } #sitelinks-mobile { display: block; position: absolute; top: -10px; left: 107px; } #sitelinks-wide ul { display: block; } #sitelinks-wide ul li { margin-right: 0%; white-space: nowrap; } .dropdown-item { display: block; width: 100%; padding: .25rem 1.5rem; clear: both; font-weight: 400; color: #212529; text-align: inherit; white-space: nowrap; background-color: transparent; border: 0; text-align: left; } .dropdown.nap-mega-menu.mega-dropdown { display: none;} li.naphelp { display: none;} } .global-nav__links { display: none; opacity: 1; transition: opacity .15s, -webkit-transform .15s; transition: opacity .15s, transform .15s; transition: opacity .15s, transform .15s, -webkit-transform .15s; } .global-nav--local .global-nav__links { -webkit-transform: translateY(0); transform: translateY(0); } .global-nav--local.menu-is-hidden .global-nav__links { opacity: 0; -webkit-transform: translateY(0.5em); transform: translateY(0.5em); } .global-nav__links ul { display: -webkit-flex; display: flex; -webkit-justify-content: flex-end; justify-content: flex-end; margin-top: 1rem; } .global-nav__links li:hover .global-nav__link>span::before, .global-nav__links li:focus-within .global-nav__link>span::before, .global-nav__links .global-nav__link.is-active>span::before, .global-nav__links .global-nav__link:hover>span::before, .global-nav__links .global-nav__link:focus>span::before { background-color: #1C143D; ; } .global-nav__links li:hover .global-nav__link>span::before::after, .global-nav__links li:focus-within .global-nav__link>span::before::after, .global-nav__links .global-nav__link.is-active>span::before::after, .global-nav__links .global-nav__link:hover>span::before::after, .global-nav__links .global-nav__link:focus>span::before::after { opacity: 1; } @media screen and (min-width: 65em) { .global-nav__links { display: block; } .global-nav__links>ul>li:last-of-type .global-nav__link { margin-right: -1.25em; } } .global-nav__link { position: relative; display: block; padding: .75em 1.5em .5em; color: rgb(34, 34, 34); color: var(--color-neutral--charcoal); border-bottom: 5px solid transparent; /*font-size: .885rem;*/ font-weight: 700; font-weight: var(--lato--bold); text-decoration: none; transition: color .15s; } .global-nav__link>span { position: relative; } .global-nav__link>span::before { content: ""; position: absolute; bottom: -.5em; left: 0; width: 100%; height: 5px; background-color: transparent; transition: background-color .15s; } .global-nav__dropdown { position: absolute; top: calc(100% + 1em); left: 50%; display: none; width: 600px; background-color: rgb(255, 255, 255); background-color: var(--color-neutral--white); box-shadow: 0 5px 15px rgba(34, 34, 34, .25); box-shadow: 0 5px 15px var(--box-shadow--charcoal); -webkit-transform: translateX(-50%); transform: translateX(-50%); z-index: 20; } .global-nav__dropdown::before { content: ""; position: absolute; top: -1em; left: 0; display: block; width: 100%; height: 1em; } .global-nav__dropdown .dropdown__left { position: relative; min-height: 260px; width: 40%; padding: 1.5rem; background: rgb(255,255,255); background: -moz-linear-gradient(180deg, rgba(255,255,255,1) 0%, rgba(73,69,104,1) 0%, rgba(0,0,0,1) 100%, rgba(195,228,228,1) 100%); background: -webkit-linear-gradient(180deg, rgba(255,255,255,1) 0%, rgba(73,69,104,1) 0%, rgba(0,0,0,1) 100%, rgba(195,228,228,1) 100%); background: linear-gradient(180deg, rgba(255,255,255,1) 0%, rgba(73,69,104,1) 0%, rgba(0,0,0,1) 100%, rgba(195,228,228,1) 100%); filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#ffffff",endColorstr="#c3e4e4",GradientType=1); } .global-nav__dropdown .dropdown__left a { display: block; color: rgb(255, 255, 255); color: var(--color-neutral--white); font-weight: 400; font-weight: var(--lato--regular); transition: color .15s; font-size: 16px; } .global-nav__dropdown .dropdown__left a:hover, .global-nav__dropdown .dropdown__left a:focus, .global-nav__dropdown .dropdown__left a:active { color: rgb(246,210,0); color: var(--color-yellow--pure); } .global-nav__dropdown .dropdown__left ul { -webkit-flex-direction: column; flex-direction: column; } .global-nav__dropdown .dropdown__left li a { padding: .5em 0; line-height: 1.4; } .global-nav__dropdown .dropdown__left--default { position: absolute; top: 0; right: 0; bottom: 0; left: 0; overflow: hidden; background-color: rgb(243, 241, 235); background-color: var(--color-brown--bone); } .global-nav__dropdown .dropdown__left--default img { width: 100%; height: auto; /* stylelint-disable max-nesting-depth */ /* stylelint-enable max-nesting-depth */ } @supports (object-fit: cover) { .global-nav__dropdown .dropdown__left--default img { height: 100%; object-fit: cover; } } .global-nav__dropdown .dropdown__right { width: 60%; padding: 1.5rem; } .global-nav__dropdown .dropdown__right .button { margin-top: 2em; text-shadow: none; } .global-nav__dropdown .dropdown__right a { color: rgb(0, 89, 145); color: var(--color-blue--bahama); text-decoration: none; transition: color .15s; } .global-nav__dropdown .dropdown__right a:hover, .global-nav__dropdown .dropdown__right a:focus { color: rgb(33,23,71); color: var(--color-blue--imperial); text-decoration: underline; } .rm-dropdown:hover::after, .rm-dropdown:focus::after { visibility: hidden; } .has-dropdown { position: relative; } .has-dropdown::after { content: ""; position: absolute; top: calc(100% + 3px); left: 50%; width: 26px; height: 15px; background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABoAAAAPCAYAAAD6Ud/mAAAABGdBTUEAALGPC/xhBQAAAldJREFUOBGtkz+oGkEQxu92NV5C0IioBEVFEQsDTzAQEAQlVYrUFinSCKnT2KpFbF+hTbBJoUUKoxIkrZVgrzaKtqIoqPjnqfEyczCi4qFFDo7dnZn9frvf7oqCyifLsniaSqfTZ2PKJZNJmfqiKB77FKP26uRTCAHa7bY4Ho/P6s1ms+z3++V7YGcTiU4ghBBgsViIm81GfHp6EnU6nbJySZJkt9t9uAemIXFqr0FgJwwBAGLdbvcr1nq93keIHfr9vgB5BB9oZ6hxaSMjALYEoRhahZDlcskMBgNvtVqfYRff8Mc+xiaTCcfd1ut1RjZf0zqCTiE4AScSRK/X81qt9hGEc7QI7GMMcwRDm9VgCugWpFqthq1W6w+AcAJhH2OYM5lMDGFgI1ODsUsIFqIVaBeutlgsPjidzp8gLJ1AqCthrlAoPCBsNpsxNdjROtwyQrAQV4eQXC7ngRv1Cw5WT8qXLeZ8Pl85m8168MzUYDyVSim+EgQLjUYjz2Qyr6PR6B/GmP1S/HIMsJc2m+2Dw+EoNxqNJWgI6/Va2O/3QqfTESKRiMChSPEVd4IQXFU8HjfGYrHfAPFdiqqNQeeVx+OJwlGUer3eZjqdChqNRoYjUGAcXrfiK0GA/iKRSJQ452/VRNXiALMGAoF3g8GgBDd2PxwORYJxi8XC5/M5vhUOr1ybz+cLWq32vZrYrTi44AiHw28qlUp5NBoddrudMkUMBoNaOvxms/kdHuOnW2L35LfbbSEUCn0Bp/7CcRwEuFXP7Hb789Vq9Qj+/tcPNV0ul4SMf+6gwxxjOvs3AAAAAElFTkSuQmCC); background-repeat: no-repeat; -webkit-transform: translateX(-50%); transform: translateX(-50%); visibility: hidden; } .has-dropdown:hover::after, .has-dropdown:focus::after { visibility: visible; } .has-dropdown:hover .global-nav__dropdown { display: -webkit-flex; display: flex; } .has-dropdown:focus-within .global-nav__dropdown { display: -webkit-flex; display: flex; } .has-dropdown:nth-last-of-type(-n+3) .global-nav__dropdown { left: auto; -webkit-transform: none; transform: none; } .has-dropdown:nth-last-of-type(3) .global-nav__dropdown { right: -200%; } .has-dropdown:nth-last-of-type(2) .global-nav__dropdown { right: -90%; } .has-dropdown:last-of-type .global-nav__dropdown { right: 5%; } .global-nav__links { z-index: 5; } header.global-nav { height: 150px; } .button.button--secondary.button--outline.button--icon { font-size: 12px; padding-right: 40px; } .button.button--icon svg { height: 2.5em; position: absolute; top: 50%; left: 60%; -webkit-transform: translate(-50%, -50%); transform: translate(-50%, -50%); width: 24px; } .controls__search, .controls__menu, .controls__nav { font-size: 12px; } .global-nav__menus--upper { display: -webkit-flex; display: flex; -webkit-justify-content: space-between; justify-content: space-between; margin-bottom: 2rem; } .global-nav__link { font-size: 16px; } .global-nav__links ul { display: -webkit-flex; display: flex; -webkit-justify-content: flex-end; justify-content: flex-end; margin-top: 5px; list-style: none; } .invert { background: #000; color: #fff; } .controls__nav .menu-off { display: inline; } .controls__nav .menu-on { display: none; } .controls__nav.invert .menu-off { display: none } .controls__nav.invert .menu-on { display: inline; } .mega-menu { background-color: rgb(243, 241, 235); background-color: var(--color-brown--bone); position: fixed; top: 0; right: 0; bottom: 0; left: 0; overflow: scroll; z-index: 9999999; } .mega-menu .mega-menu__header { display: -webkit-flex; display: flex; -webkit-justify-content: space-between; justify-content: space-between; -webkit-align-items: center; align-items: center; padding-top: 1rem; } .mega-menu__content .search-form__container { position: relative; max-width: 70em; margin: 0 auto; } #search .global-nav__logo { position: absolute; top: 1em; } .sf-top { margin-top: 50px; } #searchFormNAP { display: block; height: 100px; } a:hover, a:focus { color: #1C143D; text-decoration: underline; } #fixedfooter a:hover, #fixedfooter a:focus { color: #fff; text-decoration: underline; } .btn-blueflat.btn-searchhome.hide-xs { display: none; } .mega-menu__content { margin-top: 50px; margin-left: 7px; } .accordion__title.faq__accordion__title .flex.justify-between { display: inherit; } .global-nav__logo { margin: 0px !important; position: absolute; left: 15px; top: 25px; height: 55px; } .global-nav__link:hover { text-decoration: none; } @media screen and (min-width: 65em) { .mega-menu__content .search-form { padding: 1em 0px 3em 0em; } } @media (min-width: 45rem) { .global-nav__logo {} } @media (max-width: 480px) { .global-nav__logo { margin: 0px !important; position: absolute; left: 15px; top: 12px; } } /* Style the buttons that are used to open and close the accordion panel */ #mobile-menu button.accordion, #mobile-menu button.no-accordion { letter-spacing: 1px; text-align: left; color: #4148C2; font-size: 1.1em; font-weight: 700 !important; font-family: 'lato', helvetica, sans-serif; padding: 15px 0px 15px 0px; border-top: none !important; border-bottom: .5px solid rgb(221, 213, 199) !important; cursor: pointer; width: 100%; } .mega-menu__content .accordion { border: 0; font-weight: 700; } #mobile-menu .accordion:after { content: '\02795'; /* Unicode character for "plus" sign (+) */ font-size: 13px; color: #777; float: right; margin-left: 5px; } #mobile-menu button.active:after { content: "\2796"; /* Unicode character for "minus" sign (-) */ } /* Add a background color to the button if it is clicked on (add the .active class with JS), and when you move the mouse over it (hover) */ #mobile-menu button.active, #mobile-menu button.accordion:hover { background-color: transparent; } /* Style the accordion panel. Note: hidden by default */ #mobile-menu .panel { padding: 0 0; background-color: transparent; max-height: 0; box-shadow: 0 !important; overflow: hidden; transition: max-height 0.2s ease-out; margin: 0px !important; border: 0 !important; } #mobile-menu .panel ul li { margin: 0px 0px; padding-left: 0px; } #mobile-menu .panel ul { margin-top: 10px; margin-bottom: 10px; padding-left: 0px; } #mobile-menu .panel ul li { line-height: 2; } #mobile-menu .panel ul li a { color: #000; text-decoration: underline; cursor: pointer; } .controls__menu-mobile { text-transform: uppercase; } .controls__menu-mobile svg { fill: #222; fill: var(--color-neutral--charcoal); width: 1.75rem; height: 1.75rem; } .cart-badge { display: inline-block; text-align: center; line-height: 1; background: #7899F2; color: white; font-weight: bold; border-radius: 20px; padding: 3px 6px; margin-left: 3px; } @media screen and (min-width: 65em) { #mobile-menu .controls__menu { display: inline; } } @media screen and (max-width: 65em) { #mobile-menu .controls__menu svg { display: inline; } } /*NAP mega menu dropdown*/ li.nap-mega-menu.mega-dropdown { position: static !important; width:100% !important; } .nap-mega-menu .mega-dropdown-menu { padding: 20px 0px; box-shadow: none; -webkit-box-shadow: none; margin-top: 10px !important; box-shadow: 0 5px 15px var(--box-shadow--charcoal); transform: none !important; top: 100% !important; } .nap-mega-menu .mega-dropdown-menu > li > ul.browse-col { padding: 0; margin: 0; width: 100%; } .nap-mega-menu .mega-dropdown-menu > li > ul > li { list-style: none; } .nap-mega-menu .mega-dropdown-menu > li > ul > li > a { display: block; padding: 3px 20px; clear: both; font-weight: normal; line-height: 1.428571429; color: #4148C2; white-space: normal; font-size: 14px; text-transform: none; } .nap-mega-menu .mega-dropdown-menu > li ul > li > a:hover, .nap-mega-menu .mega-dropdown-menu > li ul > li > a:focus { text-decoration: none; color: #444; background-color: #f5f5f5; } .nap-mega-menu .mega-dropdown-menu .dropdown-header a { color: #4148C2; font-size: 14px; font-weight: 900 !important; text-align: left; text-transform: uppercase !important; padding: 0px; margin-bottom: 7px; } .nap-mega-menu .mega-dropdown-menu form { margin:3px 20px; } .nap-mega-menu .mega-dropdown-menu .form-group { margin-bottom: 3px; } .nap-mega-menu .vertical-divider { border-left: 1px solid #ccc; } .nap-mega-menu .mega-dropdown-menu > li.d-flex { display: inline-flex !important; margin-right: 0 !important; justify-content: space-between; flex-wrap: nowrap !important; flex-direction: row !important; } #nap-mega-menu-inner { justify-content: space-between; align-content:start; } #nap-mega-menu-inner li.browse-col { width: 22%; } </style> <header class="global-nav global-nav--local menu-is-hidden search-is-hidden" id="lowerMenu"> <div class="container flex justify-between"> <!--Mega menu start --> <div aria-hidden="true" class="mega-menu" id="search"> <div data-lock=""> <div class="container mega-menu__header"> <div class="global-nav__logo"> <a class="navbar-brand js-scroll-trigger" href="https://www.nationalacademies.org/"> <svg xmlns="http://www.w3.org/2000/svg" id="nasemLogo" data-name="nasemLogo" viewBox="0 0 360 66.64"> <defs/> <path d="m4.26 8.63 19.56 20.49V1.72h-3.47v18.93L.8.16v27.32h3.46V8.63zM32.55 27.48h3.83l8.15-19.2 7.93 18.99.09.21h3.82L44.62 0 32.55 27.48zM175.05 27.48h3.83l8.16-19.2 7.93 18.99.08.21h3.83L187.12 0l-12.07 27.48zM68.23 27.48h3.47V5.03h7.72V1.72H60.51v3.31h7.72v22.45zM90.08 1.72h3.46v25.76h-3.46z" class="cls-2"/> <path d="M157.14 38.67A13.39 13.39 0 1 0 143.75 52a13.21 13.21 0 0 0 13.39-13.33Zm-3.54 0a9.85 9.85 0 1 1-9.85-10 9.93 9.93 0 0 1 9.85 10Z" class="cls-2" transform="translate(-24.97 -24.05)"/> <path d="m146.56 8.63 19.56 20.49V1.72h-3.47v18.93L143.1.16v27.32h3.46V8.63zM210.95 1.72h-3.46v25.76h13.8v-3.31h-10.34V1.72z" class="cls-2"/> <path d="M67.41 63A13.13 13.13 0 0 0 54 76.37a13 13 0 0 0 13.26 13.24A13.48 13.48 0 0 0 75 87.1l.14-.1v-4.46l-.58.58a10.91 10.91 0 0 1-2.81 2 10 10 0 0 1-4.51 1.13c-6.74 0-9.77-5.78-9.77-9.89a9.82 9.82 0 0 1 9.74-10.11 10.44 10.44 0 0 1 4.18.94 12 12 0 0 1 3.17 2.27l.58.58v-4.41l-.13-.1a12.1 12.1 0 0 0-7.6-2.53ZM130 66.4a14 14 0 0 0-9.47-3h-5.45v25.75h5.42c4.25 0 6.88-.83 9.39-3a12.32 12.32 0 0 0 4.52-9.89A12.54 12.54 0 0 0 130 66.4Zm-2.39 17.26a10.37 10.37 0 0 1-7 2.18h-2.11V66.7h2.11c6.21 0 10.22 3.77 10.22 9.59a9.23 9.23 0 0 1-3.25 7.37Z" class="cls-2" transform="translate(-24.97 -24.05)"/> <path d="M119.94 65.11h15.33V61.8h-11.86v-9.07h10.38v-3.31h-10.38v-6.76h11.86v-3.32h-15.33v25.77zM201.26 65.11h15.33V61.8h-11.86v-9.07h10.37v-3.31h-10.37v-6.76h11.86v-3.32h-15.33v25.77zM158.66 58.65l-9.79-21.53-4.42 27.99h3.56l2.36-16.59 8.25 18.12 8.35-18.12 2.25 16.59h3.56l-4.14-28.03-9.98 21.57zM184.29 39.34h3.46V65.1h-3.46z" class="cls-2"/> <path d="M263.7 76.76a12.7 12.7 0 0 0-4-2.44l-2.42-1a8 8 0 0 1-2.53-1.51 2.32 2.32 0 0 1-.83-1.94c0-2.11 1.56-3.53 3.87-3.53a4.18 4.18 0 0 1 2.61.79 5.51 5.51 0 0 1 1.37 1.77l.17.34 2.8-1.85-.14-.27a7.79 7.79 0 0 0-2.56-2.91 7.16 7.16 0 0 0-4.18-1.21c-4.34 0-7.37 2.8-7.37 6.8a5.53 5.53 0 0 0 1.83 4.35 10.74 10.74 0 0 0 3.24 2l2.44 1.17a10.81 10.81 0 0 1 2.94 1.68 3.39 3.39 0 0 1 1.13 2.68A4.8 4.8 0 0 1 254 85a5.6 5.6 0 0 1-1.42-3l-.05-.39-3.4.94.07.32a9.51 9.51 0 0 0 2.43 4.61 7.73 7.73 0 0 0 5.69 2.13 7.9 7.9 0 0 0 8.28-8 6.38 6.38 0 0 0-1.9-4.85Z" class="cls-2" transform="translate(-24.97 -24.05)"/> <path d="M0 65.11h3.83l8.16-19.21 7.92 19 .09.21h3.83L12.07 37.63 0 65.11zM57.37 65.11h3.83l8.15-19.21 7.93 19 .08.21h3.83L69.44 37.63 57.37 65.11z" class="cls-2"/> <path d="M292 26.45a3.91 3.91 0 0 0-1.07-.14 2.94 2.94 0 0 0-1.74.52 1.71 1.71 0 0 0-.72 1.47 2.36 2.36 0 0 0 .63 1.7 9.45 9.45 0 0 0 1.9 1.38 15.52 15.52 0 0 1 1.45 1 4.28 4.28 0 0 1 .94 1.12 2.87 2.87 0 0 1 .38 1.51 3.57 3.57 0 0 1-.57 2 3.79 3.79 0 0 1-1.6 1.34 5.79 5.79 0 0 1-2.39.47 9.63 9.63 0 0 1-2.22-.25 4.15 4.15 0 0 1-1.41-.52l.39-2.49H287l.11 1.84a2.93 2.93 0 0 0 .92.39 5 5 0 0 0 1.16.14 3.39 3.39 0 0 0 2-.6 1.94 1.94 0 0 0 .83-1.68 2.25 2.25 0 0 0-.56-1.54 7.66 7.66 0 0 0-1.7-1.32c-.6-.39-1.1-.75-1.51-1.07a4.43 4.43 0 0 1-1-1.2 2.94 2.94 0 0 1-.43-1.59 3.08 3.08 0 0 1 .57-1.85 3.58 3.58 0 0 1 1.55-1.23 5.17 5.17 0 0 1 2.13-.43 7.84 7.84 0 0 1 2 .21 4 4 0 0 1 1.25.51l-.4 2.31h-.95l-.14-1.68a2.39 2.39 0 0 0-.83-.32ZM300.27 29.79a2.2 2.2 0 0 0-1.53.62 4.15 4.15 0 0 0-1.06 1.74 8 8 0 0 0-.38 2.55 4.11 4.11 0 0 0 .5 2.24 1.6 1.6 0 0 0 1.42.74 2.71 2.71 0 0 0 1.25-.27 5.56 5.56 0 0 0 1.15-.81l.35.64a4.42 4.42 0 0 1-1.31 1.08 3.78 3.78 0 0 1-2 .5 2.76 2.76 0 0 1-2.31-1 4.51 4.51 0 0 1-.83-2.9 7.35 7.35 0 0 1 .6-2.87 6 6 0 0 1 1.77-2.38 4.27 4.27 0 0 1 2.77-.95 5.7 5.7 0 0 1 1.06.11 3.31 3.31 0 0 1 .92.3l-.48 1.7-.29-.11a2.91 2.91 0 0 0-1.6-.93ZM306.13 37.31a.54.54 0 0 0 .08.33.22.22 0 0 0 .2.11.74.74 0 0 0 .39-.12 2.88 2.88 0 0 0 .5-.4l.23.42a3.34 3.34 0 0 1-.78.77 2 2 0 0 1-1.23.4 1.3 1.3 0 0 1-.89-.27.9.9 0 0 1-.26-.81c0-.3.18-1.43.46-3.41q.24-1.59.6-4.23l-1.09-.4.11-.6 2.49-.4.38.19-1.17 8.25Zm-.08-10.8a1 1 0 0 1-.27-.69 1.24 1.24 0 0 1 .34-.89 1.07 1.07 0 0 1 .82-.37.94.94 0 0 1 .7.27.93.93 0 0 1 .26.66 1.35 1.35 0 0 1-.33.92 1.14 1.14 0 0 1-.88.38.9.9 0 0 1-.64-.28ZM315.15 38.24a4.26 4.26 0 0 1-2.23.6 3.45 3.45 0 0 1-2-.53 3.23 3.23 0 0 1-1.15-1.41 4.83 4.83 0 0 1-.36-1.87 7.38 7.38 0 0 1 .67-3.14 5.91 5.91 0 0 1 1.92-2.33 4.33 4.33 0 0 1 2.62-.88 2.64 2.64 0 0 1 1.88.6 2.06 2.06 0 0 1 .63 1.56 2.66 2.66 0 0 1-1 2.11 6.62 6.62 0 0 1-2.37 1.28 9.48 9.48 0 0 1-2.46.45 3.85 3.85 0 0 0 .42 2.13 1.68 1.68 0 0 0 1.6.91 3.61 3.61 0 0 0 1.54-.36 4.41 4.41 0 0 0 1.39-1l.38.65a4.39 4.39 0 0 1-1.48 1.23Zm-2.41-8a4.82 4.82 0 0 0-1 1.65 7.41 7.41 0 0 0-.44 1.94 5.85 5.85 0 0 0 1.86-.39 4.34 4.34 0 0 0 1.6-1 2 2 0 0 0 .65-1.46 1.5 1.5 0 0 0-.33-1.05 1.14 1.14 0 0 0-.88-.35 2 2 0 0 0-1.46.67ZM323.42 29.32a3.62 3.62 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.6 2.93 2.93 0 0 1 .53 1.95 16.77 16.77 0 0 1-.39 2.68l-.07.4-.14.85a12.6 12.6 0 0 0-.28 2.12c0 .32.08.47.23.47a1.35 1.35 0 0 0 .82-.44l.21.46a1.7 1.7 0 0 1-.36.4 2.81 2.81 0 0 1-.7.45 1.92 1.92 0 0 1-.9.2c-.74 0-1.11-.35-1.11-1.07a14.6 14.6 0 0 1 .32-2.14l.16-.84.07-.42c.11-.63.21-1.18.28-1.66a9 9 0 0 0 .1-1.32 2 2 0 0 0-.23-1.12 1 1 0 0 0-.84-.3 2.63 2.63 0 0 0-1.57.69 6.6 6.6 0 0 0-1.48 1.67l-.86 6.34H319l1.13-8.49-.86-.42.14-.67 2.14-.39.37.2-.23 2.1a6.56 6.56 0 0 1 1.73-1.66ZM334.3 29.79a2.22 2.22 0 0 0-1.53.62 4.14 4.14 0 0 0-1 1.74 7.68 7.68 0 0 0-.38 2.55 4.11 4.11 0 0 0 .49 2.24 1.61 1.61 0 0 0 1.42.74 2.78 2.78 0 0 0 1.26-.27 5.52 5.52 0 0 0 1.14-.81l.36.64a4.56 4.56 0 0 1-1.31 1.08 3.85 3.85 0 0 1-2 .5 2.79 2.79 0 0 1-2.31-1 4.57 4.57 0 0 1-.83-2.9 7.19 7.19 0 0 1 .61-2.87 5.94 5.94 0 0 1 1.77-2.38 4.24 4.24 0 0 1 2.76-.95 5.7 5.7 0 0 1 1.06.11 3.2 3.2 0 0 1 .92.3l-.47 1.7-.3-.11a2.91 2.91 0 0 0-1.66-.93ZM343.32 38.24a4.26 4.26 0 0 1-2.23.6 3.42 3.42 0 0 1-1.95-.53A3.17 3.17 0 0 1 338 36.9a4.66 4.66 0 0 1-.37-1.87 7.53 7.53 0 0 1 .67-3.14 6 6 0 0 1 1.86-2.33 4.36 4.36 0 0 1 2.63-.88 2.65 2.65 0 0 1 1.88.6 2.06 2.06 0 0 1 .63 1.56 2.69 2.69 0 0 1-1 2.11 6.66 6.66 0 0 1-2.38 1.28 9.33 9.33 0 0 1-2.46.45 4 4 0 0 0 .42 2.13 1.68 1.68 0 0 0 1.6.91 3.65 3.65 0 0 0 1.55-.36 4.37 4.37 0 0 0 1.38-1l.39.65a4.43 4.43 0 0 1-1.48 1.23Zm-2.4-8a4.67 4.67 0 0 0-1 1.65 7 7 0 0 0-.44 1.94 5.79 5.79 0 0 0 1.85-.39 4.25 4.25 0 0 0 1.6-1 2 2 0 0 0 .65-1.46 1.5 1.5 0 0 0-.32-1.05 1.17 1.17 0 0 0-.89-.35 1.93 1.93 0 0 0-1.45.67ZM351.91 29.91a3.08 3.08 0 0 0-1.24-.24 2.15 2.15 0 0 0-1.3.35 1.18 1.18 0 0 0-.49 1 1.54 1.54 0 0 0 .4 1.06 6.17 6.17 0 0 0 1.13 1c.42.33.76.61 1 .85a3.35 3.35 0 0 1 .67.87 2.38 2.38 0 0 1 .3 1.2 2.4 2.4 0 0 1-1 2.07 4.21 4.21 0 0 1-2.53.73 7.55 7.55 0 0 1-1.57-.17 2.73 2.73 0 0 1-1-.39l.29-1.21a3.77 3.77 0 0 0 1 .5 3.94 3.94 0 0 0 1.36.24 2.47 2.47 0 0 0 1.31-.33 1.14 1.14 0 0 0 .55-1.06 1.58 1.58 0 0 0-.43-1.09 8 8 0 0 0-1.26-1 3.53 3.53 0 0 1-1.81-2.86 2.43 2.43 0 0 1 .46-1.45 2.92 2.92 0 0 1 1.25-1 4.48 4.48 0 0 1 1.75-.34 7.2 7.2 0 0 1 1.29.12 4.56 4.56 0 0 1 .92.22l-.3 1.33a1.83 1.83 0 0 0-.75-.4ZM287 50.81h8.32l-.12 2.79h-1l-.28-1.84-4-.14-.67 5 4.07-.14-.12 1.32-4-.14-.81 5.34 4.95-.21 1-2.12 1 .19-.71 3h-9.33l.07-.68 1.32-.21 1.4-11.19-1.19-.2ZM301.65 54.52a3.52 3.52 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.6 2.89 2.89 0 0 1 .53 1.94 16.85 16.85 0 0 1-.39 2.69l-.07.4-.14.84a12.68 12.68 0 0 0-.28 2.13c0 .31.07.47.23.47a1.41 1.41 0 0 0 .82-.44l.14.49a1.76 1.76 0 0 1-.37.4 3.17 3.17 0 0 1-.69.45 2 2 0 0 1-.9.2A1 1 0 0 1 303 63a14.6 14.6 0 0 1 .32-2.14l.16-.84.07-.42c.11-.64.21-1.19.28-1.66a9.18 9.18 0 0 0 .1-1.33 2 2 0 0 0-.23-1.11 1 1 0 0 0-.84-.31 2.59 2.59 0 0 0-1.57.7 6.6 6.6 0 0 0-1.48 1.67l-.86 6.33h-1.67l1.13-8.49-.86-.42.14-.66 2.14-.39.37.19-.23 2.11a6.56 6.56 0 0 1 1.68-1.71ZM313.74 66.64a4.23 4.23 0 0 1-1.77 1.2 5.86 5.86 0 0 1-2 .37 7.37 7.37 0 0 1-1.13-.09 4.89 4.89 0 0 1-.87-.22l.37-1.42a6.44 6.44 0 0 0 1 .39 4.29 4.29 0 0 0 1.22.18 2.62 2.62 0 0 0 1.66-.43 2.58 2.58 0 0 0 .71-1.62l.33-2.33a4.25 4.25 0 0 1-1.18 1 3 3 0 0 1-1.52.39 2.52 2.52 0 0 1-2.17-1 4.5 4.5 0 0 1-.72-2.68 7.33 7.33 0 0 1 .66-3 6.45 6.45 0 0 1 1.95-2.49 4.73 4.73 0 0 1 3-1 5.1 5.1 0 0 1 1 .11 5.19 5.19 0 0 1 .91.28l1.09-.39-1.58 10.63a3.73 3.73 0 0 1-.96 2.12ZM313 54.81a2.66 2.66 0 0 0-2 .85 5.43 5.43 0 0 0-1.22 2.11 7.91 7.91 0 0 0-.4 2.46 3.81 3.81 0 0 0 .42 2 1.31 1.31 0 0 0 1.17.71 2.42 2.42 0 0 0 1.37-.44 3.83 3.83 0 0 0 1-1l.93-6.28a2.13 2.13 0 0 0-1.27-.41ZM320.16 62.51a.56.56 0 0 0 .08.32.24.24 0 0 0 .2.12.8.8 0 0 0 .4-.12 4.16 4.16 0 0 0 .5-.41l.23.42a3.57 3.57 0 0 1-.78.78 2 2 0 0 1-1.24.4 1.32 1.32 0 0 1-.89-.27.92.92 0 0 1-.25-.82c0-.29.18-1.43.46-3.4.16-1.07.36-2.48.59-4.23l-1.09-.4.11-.6 2.49-.4.39.19-1.18 8.25Zm-.08-10.81a.93.93 0 0 1-.27-.68 1.25 1.25 0 0 1 .35-.89 1 1 0 0 1 .81-.37.93.93 0 0 1 .7.27.9.9 0 0 1 .27.66 1.36 1.36 0 0 1-.34.92 1.11 1.11 0 0 1-.88.37.87.87 0 0 1-.64-.28ZM328.56 54.52a3.54 3.54 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.6 2.89 2.89 0 0 1 .53 1.94 16.85 16.85 0 0 1-.39 2.69l-.07.4-.14.84a12.68 12.68 0 0 0-.28 2.13c0 .31.08.47.23.47a1.41 1.41 0 0 0 .82-.44l.23.49a1.76 1.76 0 0 1-.37.4 3.17 3.17 0 0 1-.69.45 2 2 0 0 1-.9.2 1 1 0 0 1-1.16-1.05 14.6 14.6 0 0 1 .32-2.14l.16-.84.07-.42c.11-.64.21-1.19.28-1.66a9.18 9.18 0 0 0 .1-1.33 2 2 0 0 0-.23-1.11 1 1 0 0 0-.84-.31 2.59 2.59 0 0 0-1.57.7 6.6 6.6 0 0 0-1.48 1.67l-.86 6.33h-1.67l1.13-8.49-.86-.42.14-.66 2.14-.39.37.19-.23 2.11a6.56 6.56 0 0 1 1.71-1.71ZM340.41 63.44a4.26 4.26 0 0 1-2.23.6 3.35 3.35 0 0 1-1.95-.54 3 3 0 0 1-1.15-1.4 4.66 4.66 0 0 1-.37-1.87 7.38 7.38 0 0 1 .67-3.14 5.91 5.91 0 0 1 1.86-2.33 4.29 4.29 0 0 1 2.62-.88 2.62 2.62 0 0 1 1.88.6 2 2 0 0 1 .63 1.56 2.63 2.63 0 0 1-1 2.1 6.47 6.47 0 0 1-2.37 1.28 9.38 9.38 0 0 1-2.46.46A4 4 0 0 0 337 62a1.69 1.69 0 0 0 1.6.9 3.61 3.61 0 0 0 1.54-.36 4.27 4.27 0 0 0 1.39-1l.39.65a4.68 4.68 0 0 1-1.51 1.25Zm-2.4-8a4.53 4.53 0 0 0-1 1.65 6.69 6.69 0 0 0-.44 1.94 6.13 6.13 0 0 0 1.85-.39 4.25 4.25 0 0 0 1.6-1 2 2 0 0 0 .65-1.46 1.55 1.55 0 0 0-.32-1.05 1.14 1.14 0 0 0-.89-.36 2 2 0 0 0-1.46.68ZM349.39 63.44a4.26 4.26 0 0 1-2.23.6 3.35 3.35 0 0 1-2-.54 3.09 3.09 0 0 1-1.15-1.4 4.66 4.66 0 0 1-.37-1.87 7.38 7.38 0 0 1 .68-3.14 5.91 5.91 0 0 1 1.86-2.33 4.25 4.25 0 0 1 2.62-.88 2.65 2.65 0 0 1 1.88.6 2 2 0 0 1 .63 1.56 2.66 2.66 0 0 1-1 2.1 6.51 6.51 0 0 1-2.31 1.28 9.38 9.38 0 0 1-2.46.46 3.94 3.94 0 0 0 .43 2.13 1.66 1.66 0 0 0 1.59.9 3.65 3.65 0 0 0 1.55-.36 4.34 4.34 0 0 0 1.38-1l.39.65a4.68 4.68 0 0 1-1.49 1.24Zm-2.4-8a4.67 4.67 0 0 0-1 1.65 7 7 0 0 0-.44 1.94 6.07 6.07 0 0 0 1.85-.39 4.38 4.38 0 0 0 1.61-1 2 2 0 0 0 .65-1.46 1.5 1.5 0 0 0-.33-1.05 1.14 1.14 0 0 0-.89-.36 2 2 0 0 0-1.44.68ZM356.34 55.24a4.21 4.21 0 0 1 1.13-1 2.81 2.81 0 0 1 1.4-.38.93.93 0 0 1 .56.12l-.37 1.86a1.64 1.64 0 0 0-.86-.18 2.68 2.68 0 0 0-1.48.5 3.31 3.31 0 0 0-1.14 1.4l-.8 6.24h-1.69l1.11-8.45-.88-.44.12-.65 2.07-.4.35.21-.12 1.56-.08.7a3.9 3.9 0 0 1 .68-1.09ZM362.32 62.51a.56.56 0 0 0 .08.32.23.23 0 0 0 .2.12.8.8 0 0 0 .4-.12 4.16 4.16 0 0 0 .5-.41l.22.42a3.18 3.18 0 0 1-.78.78 2 2 0 0 1-1.23.4 1.3 1.3 0 0 1-.89-.27.92.92 0 0 1-.25-.82c0-.29.17-1.43.45-3.4.17-1.07.36-2.48.6-4.23l-1.09-.4.11-.6 2.49-.4.38.19-1.17 8.25Zm-.08-10.81a.93.93 0 0 1-.24-.7 1.24 1.24 0 0 1 .34-.89 1.07 1.07 0 0 1 .82-.37.94.94 0 0 1 .7.27.93.93 0 0 1 .26.66 1.35 1.35 0 0 1-.33.92 1.12 1.12 0 0 1-.88.37.86.86 0 0 1-.67-.26ZM370.36 54.52a3.55 3.55 0 0 1 2-.64 1.91 1.91 0 0 1 1.51.6 2.89 2.89 0 0 1 .52 1.94 17.7 17.7 0 0 1-.38 2.69l-.07.4-.14.84a12.68 12.68 0 0 0-.28 2.13c0 .31.07.47.22.47a1.4 1.4 0 0 0 .83-.44l.21.46a1.76 1.76 0 0 1-.37.4 3.74 3.74 0 0 1-.69.45 2.13 2.13 0 0 1-.91.2 1 1 0 0 1-1.1-1.07 14.1 14.1 0 0 1 .31-2.14l.16-.84.07-.42c.12-.64.21-1.19.28-1.66a9.27 9.27 0 0 0 .11-1.33 1.88 1.88 0 0 0-.24-1.11 1 1 0 0 0-.83-.31 2.59 2.59 0 0 0-1.57.7 6.6 6.6 0 0 0-1.48 1.67l-.86 6.33H366l1.12-8.49-.86-.42.14-.66 2.14-.39.37.19-.23 2.11a6.72 6.72 0 0 1 1.68-1.66ZM382.46 66.64a4.23 4.23 0 0 1-1.77 1.2 5.87 5.87 0 0 1-2 .37 7.34 7.34 0 0 1-1.12-.09 4.82 4.82 0 0 1-.88-.22l.37-1.42a6.58 6.58 0 0 0 1 .39 4.24 4.24 0 0 0 1.22.18 2.57 2.57 0 0 0 1.65-.43 2.53 2.53 0 0 0 .72-1.62l.33-2.33a4.25 4.25 0 0 1-1.18 1 3.07 3.07 0 0 1-1.52.39 2.52 2.52 0 0 1-2.17-1 4.5 4.5 0 0 1-.73-2.68 7.35 7.35 0 0 1 .67-3 6.27 6.27 0 0 1 1.95-2.51 4.71 4.71 0 0 1 3-1 5.1 5.1 0 0 1 1 .11 5.47 5.47 0 0 1 .91.28l1.09-.39-1.58 10.63a3.86 3.86 0 0 1-.96 2.14Zm-.7-11.83a2.62 2.62 0 0 0-2 .85 5.31 5.31 0 0 0-1.22 2.11 7.93 7.93 0 0 0-.41 2.46 3.7 3.7 0 0 0 .43 2 1.31 1.31 0 0 0 1.17.71 2.42 2.42 0 0 0 1.37-.44 3.79 3.79 0 0 0 1-1l.93-6.28a2.08 2.08 0 0 0-1.27-.41ZM295.85 80.3q1.44-3.12 1.89-4.29h3.76l-.07.75-1.56.19-.34 11.2 1.49.21-.02.64h-4.82l.07-.68 1.6-.21.4-7.76.23-3.56L297.1 80l-4.1 9.08-.94.12-1.44-9-.55-3.33-.71 3.4-1.88 7.9 1.77.21-.07.68h-4.53l.07-.68 1.46-.21L289 77l-1.62-.19.11-.75h4.07l1.35 8.56.23 1.79.77-1.76q.49-1.22 1.94-4.35ZM308 88.64a4.35 4.35 0 0 1-2.23.59 3.38 3.38 0 0 1-2-.53 3.26 3.26 0 0 1-1.15-1.4 4.87 4.87 0 0 1-.36-1.87 7.34 7.34 0 0 1 .67-3.14 6 6 0 0 1 1.91-2.29 4.32 4.32 0 0 1 2.62-.87 2.63 2.63 0 0 1 1.88.59 2.06 2.06 0 0 1 .63 1.56 2.65 2.65 0 0 1-1 2.11 6.62 6.62 0 0 1-2.37 1.28 10 10 0 0 1-2.46.46 3.89 3.89 0 0 0 .42 2.13 1.69 1.69 0 0 0 1.6.9 3.61 3.61 0 0 0 1.54-.36 4.3 4.3 0 0 0 1.39-1l.38.65a4.52 4.52 0 0 1-1.47 1.19Zm-2.41-8a4.68 4.68 0 0 0-1 1.64 7.5 7.5 0 0 0-.44 1.94 6.11 6.11 0 0 0 1.86-.38 4.61 4.61 0 0 0 1.6-1 2 2 0 0 0 .65-1.47 1.54 1.54 0 0 0-.32-1 1.15 1.15 0 0 0-.89-.36 2 2 0 0 0-1.45.64ZM318.78 87.71c0 .29.09.44.26.44a1.42 1.42 0 0 0 .81-.44l.24.44a3.31 3.31 0 0 1-.76.71 1.9 1.9 0 0 1-1.15.36c-.69 0-1-.38-1-1.13v-.38a4.57 4.57 0 0 1-1.23 1.07 3.18 3.18 0 0 1-1.65.44 2.46 2.46 0 0 1-2.14-1 4.54 4.54 0 0 1-.72-2.68 7.24 7.24 0 0 1 .67-3 6.3 6.3 0 0 1 1.89-2.48 4.67 4.67 0 0 1 3-1 4.42 4.42 0 0 1 1.31.21l.48-3.16-1.16-.3.14-.61 2.47-.32.39.23-1.83 12.4a.59.59 0 0 0-.02.2Zm-2-7.7a2.62 2.62 0 0 0-2 .85 5.14 5.14 0 0 0-1.21 2.1 8 8 0 0 0-.4 2.47 3.83 3.83 0 0 0 .42 2 1.27 1.27 0 0 0 1.16.7 2.39 2.39 0 0 0 1.4-.46 3.7 3.7 0 0 0 1-1l.95-6.19a2.2 2.2 0 0 0-1.32-.48ZM324.44 87.71a.56.56 0 0 0 .08.32.25.25 0 0 0 .21.12.79.79 0 0 0 .39-.13 3.18 3.18 0 0 0 .5-.4l.23.42a3.35 3.35 0 0 1-.78.77 2 2 0 0 1-1.24.41 1.34 1.34 0 0 1-.89-.28.92.92 0 0 1-.25-.81c0-.29.18-1.43.46-3.41.16-1.06.36-2.47.59-4.22l-1.08-.41.1-.59 2.49-.41.39.2-1.18 8.24Zm-.07-10.81a1 1 0 0 1-.28-.68 1.29 1.29 0 0 1 .35-.9 1.1 1.1 0 0 1 .81-.37.92.92 0 0 1 1 .93 1.29 1.29 0 0 1-.34.92 1.11 1.11 0 0 1-.87.38.84.84 0 0 1-.67-.28ZM332.76 80.18a2.22 2.22 0 0 0-1.53.62 4.15 4.15 0 0 0-1.06 1.74 8 8 0 0 0-.37 2.55 4.09 4.09 0 0 0 .49 2.24 1.59 1.59 0 0 0 1.42.75 2.71 2.71 0 0 0 1.29-.28 5.56 5.56 0 0 0 1.15-.81l.35.65a4.84 4.84 0 0 1-1.3 1.08 3.94 3.94 0 0 1-2 .5 2.78 2.78 0 0 1-2.31-1 4.49 4.49 0 0 1-.83-2.9 7.35 7.35 0 0 1 .6-2.87 6.1 6.1 0 0 1 1.72-2.45 4.26 4.26 0 0 1 2.77-.94 5.66 5.66 0 0 1 1.06.1 3.31 3.31 0 0 1 .92.3l-.47 1.7-.3-.1a2.88 2.88 0 0 0-1.6-.88ZM338.88 87.71a.56.56 0 0 0 .12.29.23.23 0 0 0 .2.12.82.82 0 0 0 .4-.13 3.18 3.18 0 0 0 .5-.4l.23.42a3.56 3.56 0 0 1-.78.77 2 2 0 0 1-1.24.41 1.32 1.32 0 0 1-.89-.28.9.9 0 0 1-.25-.81c0-.29.18-1.43.46-3.41.16-1.06.36-2.47.59-4.22l-1.09-.41.11-.59 2.49-.41.39.2-1.18 8.24Zm-.08-10.81a1 1 0 0 1-.27-.68 1.29 1.29 0 0 1 .35-.9 1.07 1.07 0 0 1 .81-.37.92.92 0 0 1 1 .93 1.34 1.34 0 0 1-.34.92 1.11 1.11 0 0 1-.88.38.87.87 0 0 1-.67-.28ZM347.19 79.72a3.54 3.54 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.59 2.93 2.93 0 0 1 .53 2 16.77 16.77 0 0 1-.39 2.68l-.07.41-.14.84a12.6 12.6 0 0 0-.28 2.12c0 .32.08.48.23.48a1.38 1.38 0 0 0 .82-.44l.21.45a1.91 1.91 0 0 1-.36.41 3.11 3.11 0 0 1-.7.44 1.93 1.93 0 0 1-.9.21 1 1 0 0 1-1.11-1.07 14.24 14.24 0 0 1 .32-2.14l.16-.85.07-.42c.11-.63.21-1.18.28-1.66a9 9 0 0 0 .1-1.32 1.87 1.87 0 0 0-.23-1.11 1 1 0 0 0-.84-.31 2.66 2.66 0 0 0-1.57.69 6.48 6.48 0 0 0-1.48 1.68l-.89 6.24h-1.66l1.12-8.49-.86-.42.14-.67 2.14-.38.37.19-.23 2.1a6.52 6.52 0 0 1 1.71-1.61ZM359 88.64a4.35 4.35 0 0 1-2.23.59 3.34 3.34 0 0 1-1.95-.53 3.19 3.19 0 0 1-1.15-1.4 4.69 4.69 0 0 1-.37-1.87 7.34 7.34 0 0 1 .67-3.14 6 6 0 0 1 1.9-2.29 4.35 4.35 0 0 1 2.63-.87 2.61 2.61 0 0 1 1.87.59 2 2 0 0 1 .64 1.56 2.68 2.68 0 0 1-1 2.11 6.75 6.75 0 0 1-2.38 1.28 9.81 9.81 0 0 1-2.46.46 4.08 4.08 0 0 0 .42 2.13 1.69 1.69 0 0 0 1.6.9 3.65 3.65 0 0 0 1.55-.36 4.37 4.37 0 0 0 1.38-1l.39.65a4.57 4.57 0 0 1-1.51 1.19Zm-2.4-8a4.54 4.54 0 0 0-1 1.64 6.76 6.76 0 0 0-.44 1.94 6 6 0 0 0 1.85-.38 4.52 4.52 0 0 0 1.6-1 2 2 0 0 0 .65-1.47 1.49 1.49 0 0 0-.32-1 1.14 1.14 0 0 0-.89-.36 2 2 0 0 0-1.41.64Z" class="cls-2" transform="translate(-24.97 -24.05)"/> </svg> </a></div> <div class="global-nav__controls"><button class="controls__search" style="display: none;"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path d="M13.8 21.6a7.8 7.8 0 110-15.6 7.8 7.8 0 010 15.6zm0-2a5.8 5.8 0 100-11.6 5.8 5.8 0 000 11.6z"></path><path d="M17.899 19.325l1.414-1.414 4.322 4.322-1.414 1.414z"></path></svg></button> <button class="controls__close" onclick="toggleSearch()"><span class="menus__close-label"> Close </span> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path fill-rule="evenodd" d="M20.728 8l1.818 1.818-5.455 5.455 5.455 5.455-1.818 1.818-5.455-5.455-5.455 5.455L8 20.728l5.455-5.455L8 9.818 9.818 8l5.455 5.455z"></path></svg></button></div></div> <div class="mega-menu__content"> <div class="search-form sf-top" > <div class="search-form__container"> <label for="inputsearch"> Search for Publications Only </label> <form id="searchFormNAP" action="/search" enctype="application/x-www-form-urlencoded"> <input id="inputsearch" name="term" type="text" placeholder="Type a title, topic, ISBN, or DOI and hit enter"> <input type="hidden" name="v:project" value="uweb_proj_ext"> <button class="search-form__submit"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""> <path d="M13.8 21.6a7.8 7.8 0 110-15.6 7.8 7.8 0 010 15.6zm0-2a5.8 5.8 0 100-11.6 5.8 5.8 0 000 11.6z"></path> <path d="M17.899 19.325l1.414-1.414 4.322 4.322-1.414 1.414z"></path> </svg> </button> </form> </div> <!----></div> <div class="search-form"> <div class="search-form__container"> <label for="globalSearch"> Search All</label> <form id="searchForm" action="https://sparck.nationalacademies.org/vivisimo/cgi-bin/query-meta" enctype="application/x-www-form-urlencoded"> <input id="globalSearch" name="query" type="text" placeholder="Type anything and hit enter"> <input type="hidden" name="v:project" value="uweb_proj_ext"> <button class="search-form__submit"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""> <path d="M13.8 21.6a7.8 7.8 0 110-15.6 7.8 7.8 0 010 15.6zm0-2a5.8 5.8 0 100-11.6 5.8 5.8 0 000 11.6z"></path> <path d="M17.899 19.325l1.414-1.414 4.322 4.322-1.414 1.414z"></path> </svg> </button> </form> </div> <!----></div> <!----></div> </div> <div tabindex="0" aria-hidden="true"></div> </div> <!--Mega menu end --> <!--Mobile menu start --> <div aria-hidden="true" class="mega-menu" id="mobile-menu" style="display: none;"> <div data-lock=""> <div class="mega-menu__header"> <div class="global-nav__logo" style="padding-left: 10px;"> <a class="navbar-brand js-scroll-trigger" href="https://www.nationalacademies.org/"> <svg xmlns="http://www.w3.org/2000/svg" id="nasemLogo" data-name="nasemLogo" viewBox="0 0 360 66.64"> <defs/> <path d="m4.26 8.63 19.56 20.49V1.72h-3.47v18.93L.8.16v27.32h3.46V8.63zM32.55 27.48h3.83l8.15-19.2 7.93 18.99.09.21h3.82L44.62 0 32.55 27.48zM175.05 27.48h3.83l8.16-19.2 7.93 18.99.08.21h3.83L187.12 0l-12.07 27.48zM68.23 27.48h3.47V5.03h7.72V1.72H60.51v3.31h7.72v22.45zM90.08 1.72h3.46v25.76h-3.46z" class="cls-2"/> <path d="M157.14 38.67A13.39 13.39 0 1 0 143.75 52a13.21 13.21 0 0 0 13.39-13.33Zm-3.54 0a9.85 9.85 0 1 1-9.85-10 9.93 9.93 0 0 1 9.85 10Z" class="cls-2" transform="translate(-24.97 -24.05)"/> <path d="m146.56 8.63 19.56 20.49V1.72h-3.47v18.93L143.1.16v27.32h3.46V8.63zM210.95 1.72h-3.46v25.76h13.8v-3.31h-10.34V1.72z" class="cls-2"/> <path d="M67.41 63A13.13 13.13 0 0 0 54 76.37a13 13 0 0 0 13.26 13.24A13.48 13.48 0 0 0 75 87.1l.14-.1v-4.46l-.58.58a10.91 10.91 0 0 1-2.81 2 10 10 0 0 1-4.51 1.13c-6.74 0-9.77-5.78-9.77-9.89a9.82 9.82 0 0 1 9.74-10.11 10.44 10.44 0 0 1 4.18.94 12 12 0 0 1 3.17 2.27l.58.58v-4.41l-.13-.1a12.1 12.1 0 0 0-7.6-2.53ZM130 66.4a14 14 0 0 0-9.47-3h-5.45v25.75h5.42c4.25 0 6.88-.83 9.39-3a12.32 12.32 0 0 0 4.52-9.89A12.54 12.54 0 0 0 130 66.4Zm-2.39 17.26a10.37 10.37 0 0 1-7 2.18h-2.11V66.7h2.11c6.21 0 10.22 3.77 10.22 9.59a9.23 9.23 0 0 1-3.25 7.37Z" class="cls-2" transform="translate(-24.97 -24.05)"/> <path d="M119.94 65.11h15.33V61.8h-11.86v-9.07h10.38v-3.31h-10.38v-6.76h11.86v-3.32h-15.33v25.77zM201.26 65.11h15.33V61.8h-11.86v-9.07h10.37v-3.31h-10.37v-6.76h11.86v-3.32h-15.33v25.77zM158.66 58.65l-9.79-21.53-4.42 27.99h3.56l2.36-16.59 8.25 18.12 8.35-18.12 2.25 16.59h3.56l-4.14-28.03-9.98 21.57zM184.29 39.34h3.46V65.1h-3.46z" class="cls-2"/> <path d="M263.7 76.76a12.7 12.7 0 0 0-4-2.44l-2.42-1a8 8 0 0 1-2.53-1.51 2.32 2.32 0 0 1-.83-1.94c0-2.11 1.56-3.53 3.87-3.53a4.18 4.18 0 0 1 2.61.79 5.51 5.51 0 0 1 1.37 1.77l.17.34 2.8-1.85-.14-.27a7.79 7.79 0 0 0-2.56-2.91 7.16 7.16 0 0 0-4.18-1.21c-4.34 0-7.37 2.8-7.37 6.8a5.53 5.53 0 0 0 1.83 4.35 10.74 10.74 0 0 0 3.24 2l2.44 1.17a10.81 10.81 0 0 1 2.94 1.68 3.39 3.39 0 0 1 1.13 2.68A4.8 4.8 0 0 1 254 85a5.6 5.6 0 0 1-1.42-3l-.05-.39-3.4.94.07.32a9.51 9.51 0 0 0 2.43 4.61 7.73 7.73 0 0 0 5.69 2.13 7.9 7.9 0 0 0 8.28-8 6.38 6.38 0 0 0-1.9-4.85Z" class="cls-2" transform="translate(-24.97 -24.05)"/> <path d="M0 65.11h3.83l8.16-19.21 7.92 19 .09.21h3.83L12.07 37.63 0 65.11zM57.37 65.11h3.83l8.15-19.21 7.93 19 .08.21h3.83L69.44 37.63 57.37 65.11z" class="cls-2"/> <path d="M292 26.45a3.91 3.91 0 0 0-1.07-.14 2.94 2.94 0 0 0-1.74.52 1.71 1.71 0 0 0-.72 1.47 2.36 2.36 0 0 0 .63 1.7 9.45 9.45 0 0 0 1.9 1.38 15.52 15.52 0 0 1 1.45 1 4.28 4.28 0 0 1 .94 1.12 2.87 2.87 0 0 1 .38 1.51 3.57 3.57 0 0 1-.57 2 3.79 3.79 0 0 1-1.6 1.34 5.79 5.79 0 0 1-2.39.47 9.63 9.63 0 0 1-2.22-.25 4.15 4.15 0 0 1-1.41-.52l.39-2.49H287l.11 1.84a2.93 2.93 0 0 0 .92.39 5 5 0 0 0 1.16.14 3.39 3.39 0 0 0 2-.6 1.94 1.94 0 0 0 .83-1.68 2.25 2.25 0 0 0-.56-1.54 7.66 7.66 0 0 0-1.7-1.32c-.6-.39-1.1-.75-1.51-1.07a4.43 4.43 0 0 1-1-1.2 2.94 2.94 0 0 1-.43-1.59 3.08 3.08 0 0 1 .57-1.85 3.58 3.58 0 0 1 1.55-1.23 5.17 5.17 0 0 1 2.13-.43 7.84 7.84 0 0 1 2 .21 4 4 0 0 1 1.25.51l-.4 2.31h-.95l-.14-1.68a2.39 2.39 0 0 0-.83-.32ZM300.27 29.79a2.2 2.2 0 0 0-1.53.62 4.15 4.15 0 0 0-1.06 1.74 8 8 0 0 0-.38 2.55 4.11 4.11 0 0 0 .5 2.24 1.6 1.6 0 0 0 1.42.74 2.71 2.71 0 0 0 1.25-.27 5.56 5.56 0 0 0 1.15-.81l.35.64a4.42 4.42 0 0 1-1.31 1.08 3.78 3.78 0 0 1-2 .5 2.76 2.76 0 0 1-2.31-1 4.51 4.51 0 0 1-.83-2.9 7.35 7.35 0 0 1 .6-2.87 6 6 0 0 1 1.77-2.38 4.27 4.27 0 0 1 2.77-.95 5.7 5.7 0 0 1 1.06.11 3.31 3.31 0 0 1 .92.3l-.48 1.7-.29-.11a2.91 2.91 0 0 0-1.6-.93ZM306.13 37.31a.54.54 0 0 0 .08.33.22.22 0 0 0 .2.11.74.74 0 0 0 .39-.12 2.88 2.88 0 0 0 .5-.4l.23.42a3.34 3.34 0 0 1-.78.77 2 2 0 0 1-1.23.4 1.3 1.3 0 0 1-.89-.27.9.9 0 0 1-.26-.81c0-.3.18-1.43.46-3.41q.24-1.59.6-4.23l-1.09-.4.11-.6 2.49-.4.38.19-1.17 8.25Zm-.08-10.8a1 1 0 0 1-.27-.69 1.24 1.24 0 0 1 .34-.89 1.07 1.07 0 0 1 .82-.37.94.94 0 0 1 .7.27.93.93 0 0 1 .26.66 1.35 1.35 0 0 1-.33.92 1.14 1.14 0 0 1-.88.38.9.9 0 0 1-.64-.28ZM315.15 38.24a4.26 4.26 0 0 1-2.23.6 3.45 3.45 0 0 1-2-.53 3.23 3.23 0 0 1-1.15-1.41 4.83 4.83 0 0 1-.36-1.87 7.38 7.38 0 0 1 .67-3.14 5.91 5.91 0 0 1 1.92-2.33 4.33 4.33 0 0 1 2.62-.88 2.64 2.64 0 0 1 1.88.6 2.06 2.06 0 0 1 .63 1.56 2.66 2.66 0 0 1-1 2.11 6.62 6.62 0 0 1-2.37 1.28 9.48 9.48 0 0 1-2.46.45 3.85 3.85 0 0 0 .42 2.13 1.68 1.68 0 0 0 1.6.91 3.61 3.61 0 0 0 1.54-.36 4.41 4.41 0 0 0 1.39-1l.38.65a4.39 4.39 0 0 1-1.48 1.23Zm-2.41-8a4.82 4.82 0 0 0-1 1.65 7.41 7.41 0 0 0-.44 1.94 5.85 5.85 0 0 0 1.86-.39 4.34 4.34 0 0 0 1.6-1 2 2 0 0 0 .65-1.46 1.5 1.5 0 0 0-.33-1.05 1.14 1.14 0 0 0-.88-.35 2 2 0 0 0-1.46.67ZM323.42 29.32a3.62 3.62 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.6 2.93 2.93 0 0 1 .53 1.95 16.77 16.77 0 0 1-.39 2.68l-.07.4-.14.85a12.6 12.6 0 0 0-.28 2.12c0 .32.08.47.23.47a1.35 1.35 0 0 0 .82-.44l.21.46a1.7 1.7 0 0 1-.36.4 2.81 2.81 0 0 1-.7.45 1.92 1.92 0 0 1-.9.2c-.74 0-1.11-.35-1.11-1.07a14.6 14.6 0 0 1 .32-2.14l.16-.84.07-.42c.11-.63.21-1.18.28-1.66a9 9 0 0 0 .1-1.32 2 2 0 0 0-.23-1.12 1 1 0 0 0-.84-.3 2.63 2.63 0 0 0-1.57.69 6.6 6.6 0 0 0-1.48 1.67l-.86 6.34H319l1.13-8.49-.86-.42.14-.67 2.14-.39.37.2-.23 2.1a6.56 6.56 0 0 1 1.73-1.66ZM334.3 29.79a2.22 2.22 0 0 0-1.53.62 4.14 4.14 0 0 0-1 1.74 7.68 7.68 0 0 0-.38 2.55 4.11 4.11 0 0 0 .49 2.24 1.61 1.61 0 0 0 1.42.74 2.78 2.78 0 0 0 1.26-.27 5.52 5.52 0 0 0 1.14-.81l.36.64a4.56 4.56 0 0 1-1.31 1.08 3.85 3.85 0 0 1-2 .5 2.79 2.79 0 0 1-2.31-1 4.57 4.57 0 0 1-.83-2.9 7.19 7.19 0 0 1 .61-2.87 5.94 5.94 0 0 1 1.77-2.38 4.24 4.24 0 0 1 2.76-.95 5.7 5.7 0 0 1 1.06.11 3.2 3.2 0 0 1 .92.3l-.47 1.7-.3-.11a2.91 2.91 0 0 0-1.66-.93ZM343.32 38.24a4.26 4.26 0 0 1-2.23.6 3.42 3.42 0 0 1-1.95-.53A3.17 3.17 0 0 1 338 36.9a4.66 4.66 0 0 1-.37-1.87 7.53 7.53 0 0 1 .67-3.14 6 6 0 0 1 1.86-2.33 4.36 4.36 0 0 1 2.63-.88 2.65 2.65 0 0 1 1.88.6 2.06 2.06 0 0 1 .63 1.56 2.69 2.69 0 0 1-1 2.11 6.66 6.66 0 0 1-2.38 1.28 9.33 9.33 0 0 1-2.46.45 4 4 0 0 0 .42 2.13 1.68 1.68 0 0 0 1.6.91 3.65 3.65 0 0 0 1.55-.36 4.37 4.37 0 0 0 1.38-1l.39.65a4.43 4.43 0 0 1-1.48 1.23Zm-2.4-8a4.67 4.67 0 0 0-1 1.65 7 7 0 0 0-.44 1.94 5.79 5.79 0 0 0 1.85-.39 4.25 4.25 0 0 0 1.6-1 2 2 0 0 0 .65-1.46 1.5 1.5 0 0 0-.32-1.05 1.17 1.17 0 0 0-.89-.35 1.93 1.93 0 0 0-1.45.67ZM351.91 29.91a3.08 3.08 0 0 0-1.24-.24 2.15 2.15 0 0 0-1.3.35 1.18 1.18 0 0 0-.49 1 1.54 1.54 0 0 0 .4 1.06 6.17 6.17 0 0 0 1.13 1c.42.33.76.61 1 .85a3.35 3.35 0 0 1 .67.87 2.38 2.38 0 0 1 .3 1.2 2.4 2.4 0 0 1-1 2.07 4.21 4.21 0 0 1-2.53.73 7.55 7.55 0 0 1-1.57-.17 2.73 2.73 0 0 1-1-.39l.29-1.21a3.77 3.77 0 0 0 1 .5 3.94 3.94 0 0 0 1.36.24 2.47 2.47 0 0 0 1.31-.33 1.14 1.14 0 0 0 .55-1.06 1.58 1.58 0 0 0-.43-1.09 8 8 0 0 0-1.26-1 3.53 3.53 0 0 1-1.81-2.86 2.43 2.43 0 0 1 .46-1.45 2.92 2.92 0 0 1 1.25-1 4.48 4.48 0 0 1 1.75-.34 7.2 7.2 0 0 1 1.29.12 4.56 4.56 0 0 1 .92.22l-.3 1.33a1.83 1.83 0 0 0-.75-.4ZM287 50.81h8.32l-.12 2.79h-1l-.28-1.84-4-.14-.67 5 4.07-.14-.12 1.32-4-.14-.81 5.34 4.95-.21 1-2.12 1 .19-.71 3h-9.33l.07-.68 1.32-.21 1.4-11.19-1.19-.2ZM301.65 54.52a3.52 3.52 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.6 2.89 2.89 0 0 1 .53 1.94 16.85 16.85 0 0 1-.39 2.69l-.07.4-.14.84a12.68 12.68 0 0 0-.28 2.13c0 .31.07.47.23.47a1.41 1.41 0 0 0 .82-.44l.14.49a1.76 1.76 0 0 1-.37.4 3.17 3.17 0 0 1-.69.45 2 2 0 0 1-.9.2A1 1 0 0 1 303 63a14.6 14.6 0 0 1 .32-2.14l.16-.84.07-.42c.11-.64.21-1.19.28-1.66a9.18 9.18 0 0 0 .1-1.33 2 2 0 0 0-.23-1.11 1 1 0 0 0-.84-.31 2.59 2.59 0 0 0-1.57.7 6.6 6.6 0 0 0-1.48 1.67l-.86 6.33h-1.67l1.13-8.49-.86-.42.14-.66 2.14-.39.37.19-.23 2.11a6.56 6.56 0 0 1 1.68-1.71ZM313.74 66.64a4.23 4.23 0 0 1-1.77 1.2 5.86 5.86 0 0 1-2 .37 7.37 7.37 0 0 1-1.13-.09 4.89 4.89 0 0 1-.87-.22l.37-1.42a6.44 6.44 0 0 0 1 .39 4.29 4.29 0 0 0 1.22.18 2.62 2.62 0 0 0 1.66-.43 2.58 2.58 0 0 0 .71-1.62l.33-2.33a4.25 4.25 0 0 1-1.18 1 3 3 0 0 1-1.52.39 2.52 2.52 0 0 1-2.17-1 4.5 4.5 0 0 1-.72-2.68 7.33 7.33 0 0 1 .66-3 6.45 6.45 0 0 1 1.95-2.49 4.73 4.73 0 0 1 3-1 5.1 5.1 0 0 1 1 .11 5.19 5.19 0 0 1 .91.28l1.09-.39-1.58 10.63a3.73 3.73 0 0 1-.96 2.12ZM313 54.81a2.66 2.66 0 0 0-2 .85 5.43 5.43 0 0 0-1.22 2.11 7.91 7.91 0 0 0-.4 2.46 3.81 3.81 0 0 0 .42 2 1.31 1.31 0 0 0 1.17.71 2.42 2.42 0 0 0 1.37-.44 3.83 3.83 0 0 0 1-1l.93-6.28a2.13 2.13 0 0 0-1.27-.41ZM320.16 62.51a.56.56 0 0 0 .08.32.24.24 0 0 0 .2.12.8.8 0 0 0 .4-.12 4.16 4.16 0 0 0 .5-.41l.23.42a3.57 3.57 0 0 1-.78.78 2 2 0 0 1-1.24.4 1.32 1.32 0 0 1-.89-.27.92.92 0 0 1-.25-.82c0-.29.18-1.43.46-3.4.16-1.07.36-2.48.59-4.23l-1.09-.4.11-.6 2.49-.4.39.19-1.18 8.25Zm-.08-10.81a.93.93 0 0 1-.27-.68 1.25 1.25 0 0 1 .35-.89 1 1 0 0 1 .81-.37.93.93 0 0 1 .7.27.9.9 0 0 1 .27.66 1.36 1.36 0 0 1-.34.92 1.11 1.11 0 0 1-.88.37.87.87 0 0 1-.64-.28ZM328.56 54.52a3.54 3.54 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.6 2.89 2.89 0 0 1 .53 1.94 16.85 16.85 0 0 1-.39 2.69l-.07.4-.14.84a12.68 12.68 0 0 0-.28 2.13c0 .31.08.47.23.47a1.41 1.41 0 0 0 .82-.44l.23.49a1.76 1.76 0 0 1-.37.4 3.17 3.17 0 0 1-.69.45 2 2 0 0 1-.9.2 1 1 0 0 1-1.16-1.05 14.6 14.6 0 0 1 .32-2.14l.16-.84.07-.42c.11-.64.21-1.19.28-1.66a9.18 9.18 0 0 0 .1-1.33 2 2 0 0 0-.23-1.11 1 1 0 0 0-.84-.31 2.59 2.59 0 0 0-1.57.7 6.6 6.6 0 0 0-1.48 1.67l-.86 6.33h-1.67l1.13-8.49-.86-.42.14-.66 2.14-.39.37.19-.23 2.11a6.56 6.56 0 0 1 1.71-1.71ZM340.41 63.44a4.26 4.26 0 0 1-2.23.6 3.35 3.35 0 0 1-1.95-.54 3 3 0 0 1-1.15-1.4 4.66 4.66 0 0 1-.37-1.87 7.38 7.38 0 0 1 .67-3.14 5.91 5.91 0 0 1 1.86-2.33 4.29 4.29 0 0 1 2.62-.88 2.62 2.62 0 0 1 1.88.6 2 2 0 0 1 .63 1.56 2.63 2.63 0 0 1-1 2.1 6.47 6.47 0 0 1-2.37 1.28 9.38 9.38 0 0 1-2.46.46A4 4 0 0 0 337 62a1.69 1.69 0 0 0 1.6.9 3.61 3.61 0 0 0 1.54-.36 4.27 4.27 0 0 0 1.39-1l.39.65a4.68 4.68 0 0 1-1.51 1.25Zm-2.4-8a4.53 4.53 0 0 0-1 1.65 6.69 6.69 0 0 0-.44 1.94 6.13 6.13 0 0 0 1.85-.39 4.25 4.25 0 0 0 1.6-1 2 2 0 0 0 .65-1.46 1.55 1.55 0 0 0-.32-1.05 1.14 1.14 0 0 0-.89-.36 2 2 0 0 0-1.46.68ZM349.39 63.44a4.26 4.26 0 0 1-2.23.6 3.35 3.35 0 0 1-2-.54 3.09 3.09 0 0 1-1.15-1.4 4.66 4.66 0 0 1-.37-1.87 7.38 7.38 0 0 1 .68-3.14 5.91 5.91 0 0 1 1.86-2.33 4.25 4.25 0 0 1 2.62-.88 2.65 2.65 0 0 1 1.88.6 2 2 0 0 1 .63 1.56 2.66 2.66 0 0 1-1 2.1 6.51 6.51 0 0 1-2.31 1.28 9.38 9.38 0 0 1-2.46.46 3.94 3.94 0 0 0 .43 2.13 1.66 1.66 0 0 0 1.59.9 3.65 3.65 0 0 0 1.55-.36 4.34 4.34 0 0 0 1.38-1l.39.65a4.68 4.68 0 0 1-1.49 1.24Zm-2.4-8a4.67 4.67 0 0 0-1 1.65 7 7 0 0 0-.44 1.94 6.07 6.07 0 0 0 1.85-.39 4.38 4.38 0 0 0 1.61-1 2 2 0 0 0 .65-1.46 1.5 1.5 0 0 0-.33-1.05 1.14 1.14 0 0 0-.89-.36 2 2 0 0 0-1.44.68ZM356.34 55.24a4.21 4.21 0 0 1 1.13-1 2.81 2.81 0 0 1 1.4-.38.93.93 0 0 1 .56.12l-.37 1.86a1.64 1.64 0 0 0-.86-.18 2.68 2.68 0 0 0-1.48.5 3.31 3.31 0 0 0-1.14 1.4l-.8 6.24h-1.69l1.11-8.45-.88-.44.12-.65 2.07-.4.35.21-.12 1.56-.08.7a3.9 3.9 0 0 1 .68-1.09ZM362.32 62.51a.56.56 0 0 0 .08.32.23.23 0 0 0 .2.12.8.8 0 0 0 .4-.12 4.16 4.16 0 0 0 .5-.41l.22.42a3.18 3.18 0 0 1-.78.78 2 2 0 0 1-1.23.4 1.3 1.3 0 0 1-.89-.27.92.92 0 0 1-.25-.82c0-.29.17-1.43.45-3.4.17-1.07.36-2.48.6-4.23l-1.09-.4.11-.6 2.49-.4.38.19-1.17 8.25Zm-.08-10.81a.93.93 0 0 1-.24-.7 1.24 1.24 0 0 1 .34-.89 1.07 1.07 0 0 1 .82-.37.94.94 0 0 1 .7.27.93.93 0 0 1 .26.66 1.35 1.35 0 0 1-.33.92 1.12 1.12 0 0 1-.88.37.86.86 0 0 1-.67-.26ZM370.36 54.52a3.55 3.55 0 0 1 2-.64 1.91 1.91 0 0 1 1.51.6 2.89 2.89 0 0 1 .52 1.94 17.7 17.7 0 0 1-.38 2.69l-.07.4-.14.84a12.68 12.68 0 0 0-.28 2.13c0 .31.07.47.22.47a1.4 1.4 0 0 0 .83-.44l.21.46a1.76 1.76 0 0 1-.37.4 3.74 3.74 0 0 1-.69.45 2.13 2.13 0 0 1-.91.2 1 1 0 0 1-1.1-1.07 14.1 14.1 0 0 1 .31-2.14l.16-.84.07-.42c.12-.64.21-1.19.28-1.66a9.27 9.27 0 0 0 .11-1.33 1.88 1.88 0 0 0-.24-1.11 1 1 0 0 0-.83-.31 2.59 2.59 0 0 0-1.57.7 6.6 6.6 0 0 0-1.48 1.67l-.86 6.33H366l1.12-8.49-.86-.42.14-.66 2.14-.39.37.19-.23 2.11a6.72 6.72 0 0 1 1.68-1.66ZM382.46 66.64a4.23 4.23 0 0 1-1.77 1.2 5.87 5.87 0 0 1-2 .37 7.34 7.34 0 0 1-1.12-.09 4.82 4.82 0 0 1-.88-.22l.37-1.42a6.58 6.58 0 0 0 1 .39 4.24 4.24 0 0 0 1.22.18 2.57 2.57 0 0 0 1.65-.43 2.53 2.53 0 0 0 .72-1.62l.33-2.33a4.25 4.25 0 0 1-1.18 1 3.07 3.07 0 0 1-1.52.39 2.52 2.52 0 0 1-2.17-1 4.5 4.5 0 0 1-.73-2.68 7.35 7.35 0 0 1 .67-3 6.27 6.27 0 0 1 1.95-2.51 4.71 4.71 0 0 1 3-1 5.1 5.1 0 0 1 1 .11 5.47 5.47 0 0 1 .91.28l1.09-.39-1.58 10.63a3.86 3.86 0 0 1-.96 2.14Zm-.7-11.83a2.62 2.62 0 0 0-2 .85 5.31 5.31 0 0 0-1.22 2.11 7.93 7.93 0 0 0-.41 2.46 3.7 3.7 0 0 0 .43 2 1.31 1.31 0 0 0 1.17.71 2.42 2.42 0 0 0 1.37-.44 3.79 3.79 0 0 0 1-1l.93-6.28a2.08 2.08 0 0 0-1.27-.41ZM295.85 80.3q1.44-3.12 1.89-4.29h3.76l-.07.75-1.56.19-.34 11.2 1.49.21-.02.64h-4.82l.07-.68 1.6-.21.4-7.76.23-3.56L297.1 80l-4.1 9.08-.94.12-1.44-9-.55-3.33-.71 3.4-1.88 7.9 1.77.21-.07.68h-4.53l.07-.68 1.46-.21L289 77l-1.62-.19.11-.75h4.07l1.35 8.56.23 1.79.77-1.76q.49-1.22 1.94-4.35ZM308 88.64a4.35 4.35 0 0 1-2.23.59 3.38 3.38 0 0 1-2-.53 3.26 3.26 0 0 1-1.15-1.4 4.87 4.87 0 0 1-.36-1.87 7.34 7.34 0 0 1 .67-3.14 6 6 0 0 1 1.91-2.29 4.32 4.32 0 0 1 2.62-.87 2.63 2.63 0 0 1 1.88.59 2.06 2.06 0 0 1 .63 1.56 2.65 2.65 0 0 1-1 2.11 6.62 6.62 0 0 1-2.37 1.28 10 10 0 0 1-2.46.46 3.89 3.89 0 0 0 .42 2.13 1.69 1.69 0 0 0 1.6.9 3.61 3.61 0 0 0 1.54-.36 4.3 4.3 0 0 0 1.39-1l.38.65a4.52 4.52 0 0 1-1.47 1.19Zm-2.41-8a4.68 4.68 0 0 0-1 1.64 7.5 7.5 0 0 0-.44 1.94 6.11 6.11 0 0 0 1.86-.38 4.61 4.61 0 0 0 1.6-1 2 2 0 0 0 .65-1.47 1.54 1.54 0 0 0-.32-1 1.15 1.15 0 0 0-.89-.36 2 2 0 0 0-1.45.64ZM318.78 87.71c0 .29.09.44.26.44a1.42 1.42 0 0 0 .81-.44l.24.44a3.31 3.31 0 0 1-.76.71 1.9 1.9 0 0 1-1.15.36c-.69 0-1-.38-1-1.13v-.38a4.57 4.57 0 0 1-1.23 1.07 3.18 3.18 0 0 1-1.65.44 2.46 2.46 0 0 1-2.14-1 4.54 4.54 0 0 1-.72-2.68 7.24 7.24 0 0 1 .67-3 6.3 6.3 0 0 1 1.89-2.48 4.67 4.67 0 0 1 3-1 4.42 4.42 0 0 1 1.31.21l.48-3.16-1.16-.3.14-.61 2.47-.32.39.23-1.83 12.4a.59.59 0 0 0-.02.2Zm-2-7.7a2.62 2.62 0 0 0-2 .85 5.14 5.14 0 0 0-1.21 2.1 8 8 0 0 0-.4 2.47 3.83 3.83 0 0 0 .42 2 1.27 1.27 0 0 0 1.16.7 2.39 2.39 0 0 0 1.4-.46 3.7 3.7 0 0 0 1-1l.95-6.19a2.2 2.2 0 0 0-1.32-.48ZM324.44 87.71a.56.56 0 0 0 .08.32.25.25 0 0 0 .21.12.79.79 0 0 0 .39-.13 3.18 3.18 0 0 0 .5-.4l.23.42a3.35 3.35 0 0 1-.78.77 2 2 0 0 1-1.24.41 1.34 1.34 0 0 1-.89-.28.92.92 0 0 1-.25-.81c0-.29.18-1.43.46-3.41.16-1.06.36-2.47.59-4.22l-1.08-.41.1-.59 2.49-.41.39.2-1.18 8.24Zm-.07-10.81a1 1 0 0 1-.28-.68 1.29 1.29 0 0 1 .35-.9 1.1 1.1 0 0 1 .81-.37.92.92 0 0 1 1 .93 1.29 1.29 0 0 1-.34.92 1.11 1.11 0 0 1-.87.38.84.84 0 0 1-.67-.28ZM332.76 80.18a2.22 2.22 0 0 0-1.53.62 4.15 4.15 0 0 0-1.06 1.74 8 8 0 0 0-.37 2.55 4.09 4.09 0 0 0 .49 2.24 1.59 1.59 0 0 0 1.42.75 2.71 2.71 0 0 0 1.29-.28 5.56 5.56 0 0 0 1.15-.81l.35.65a4.84 4.84 0 0 1-1.3 1.08 3.94 3.94 0 0 1-2 .5 2.78 2.78 0 0 1-2.31-1 4.49 4.49 0 0 1-.83-2.9 7.35 7.35 0 0 1 .6-2.87 6.1 6.1 0 0 1 1.72-2.45 4.26 4.26 0 0 1 2.77-.94 5.66 5.66 0 0 1 1.06.1 3.31 3.31 0 0 1 .92.3l-.47 1.7-.3-.1a2.88 2.88 0 0 0-1.6-.88ZM338.88 87.71a.56.56 0 0 0 .12.29.23.23 0 0 0 .2.12.82.82 0 0 0 .4-.13 3.18 3.18 0 0 0 .5-.4l.23.42a3.56 3.56 0 0 1-.78.77 2 2 0 0 1-1.24.41 1.32 1.32 0 0 1-.89-.28.9.9 0 0 1-.25-.81c0-.29.18-1.43.46-3.41.16-1.06.36-2.47.59-4.22l-1.09-.41.11-.59 2.49-.41.39.2-1.18 8.24Zm-.08-10.81a1 1 0 0 1-.27-.68 1.29 1.29 0 0 1 .35-.9 1.07 1.07 0 0 1 .81-.37.92.92 0 0 1 1 .93 1.34 1.34 0 0 1-.34.92 1.11 1.11 0 0 1-.88.38.87.87 0 0 1-.67-.28ZM347.19 79.72a3.54 3.54 0 0 1 2-.64 1.89 1.89 0 0 1 1.51.59 2.93 2.93 0 0 1 .53 2 16.77 16.77 0 0 1-.39 2.68l-.07.41-.14.84a12.6 12.6 0 0 0-.28 2.12c0 .32.08.48.23.48a1.38 1.38 0 0 0 .82-.44l.21.45a1.91 1.91 0 0 1-.36.41 3.11 3.11 0 0 1-.7.44 1.93 1.93 0 0 1-.9.21 1 1 0 0 1-1.11-1.07 14.24 14.24 0 0 1 .32-2.14l.16-.85.07-.42c.11-.63.21-1.18.28-1.66a9 9 0 0 0 .1-1.32 1.87 1.87 0 0 0-.23-1.11 1 1 0 0 0-.84-.31 2.66 2.66 0 0 0-1.57.69 6.48 6.48 0 0 0-1.48 1.68l-.89 6.24h-1.66l1.12-8.49-.86-.42.14-.67 2.14-.38.37.19-.23 2.1a6.52 6.52 0 0 1 1.71-1.61ZM359 88.64a4.35 4.35 0 0 1-2.23.59 3.34 3.34 0 0 1-1.95-.53 3.19 3.19 0 0 1-1.15-1.4 4.69 4.69 0 0 1-.37-1.87 7.34 7.34 0 0 1 .67-3.14 6 6 0 0 1 1.9-2.29 4.35 4.35 0 0 1 2.63-.87 2.61 2.61 0 0 1 1.87.59 2 2 0 0 1 .64 1.56 2.68 2.68 0 0 1-1 2.11 6.75 6.75 0 0 1-2.38 1.28 9.81 9.81 0 0 1-2.46.46 4.08 4.08 0 0 0 .42 2.13 1.69 1.69 0 0 0 1.6.9 3.65 3.65 0 0 0 1.55-.36 4.37 4.37 0 0 0 1.38-1l.39.65a4.57 4.57 0 0 1-1.51 1.19Zm-2.4-8a4.54 4.54 0 0 0-1 1.64 6.76 6.76 0 0 0-.44 1.94 6 6 0 0 0 1.85-.38 4.52 4.52 0 0 0 1.6-1 2 2 0 0 0 .65-1.47 1.49 1.49 0 0 0-.32-1 1.14 1.14 0 0 0-.89-.36 2 2 0 0 0-1.41.64Z" class="cls-2" transform="translate(-24.97 -24.05)"/> </svg> </a> </div> <div class="global-nav__controls"><button class="controls__menu"> Close <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path fill-rule="evenodd" d="M20.728 8l1.818 1.818-5.455 5.455 5.455 5.455-1.818 1.818-5.455-5.455-5.455 5.455L8 20.728l5.455-5.455L8 9.818 9.818 8l5.455 5.455z"></path></svg> </button> </div></div> <div class="mega-menu__content"><div> <button class="accordion">About Us</button> <div class="panel"><ul><li><a href="https://www.nationalacademies.org/about"> About Us </a></li><li><a href="https://www.nationalacademies.org/about/organization"> Our Program Divisions </a></li><li><a href="https://www.nationalacademies.org/about/the-national-academies"> Our Three Academies </a></li><li><a href="https://www.nationalacademies.org/about/leadership"> Leadership </a></li><li><a href="https://www.nationalacademies.org/newsroom"> Newsroom </a></li><li><a href="https://www.nationalacademies.org/ocga"> Government Affairs </a></li><li><a href="https://www.nationalacademies.org/giving"> Giving </a></li><li><a href="https://www.nationalacademies.org/about/institutional-policies-and-procedures/statement-on-diversity-and-inclusion"> Statement on Diversity and Inclusion </a></li></ul> </div> <button class="no-accordion"><a href="https://www.nationalacademies.org/events">Events</a></button> <button class="accordion">Our Work</button> <div class="panel"><ul><li><a href="https://www.nationalacademies.org/our-work"> Our Work </a></li><li><a href="https://www.nationalacademies.org/about/our-study-process"> Our Study Process </a></li><li><a href="https://www.nationalacademies.org/about/institutional-policies-and-procedures/conflict-of-interest-policies-and-procedures"> Conflict of Interest Policies and Procedures </a></li><li><a href="https://www.nationalacademies.org/about/institutional-policies-and-procedures/project-comments-and-information"> Project Comments and Information </a></li></ul> </div> <button class="accordion">Publications</button> <div class="panel"> <ul><li><a href="https://nap.nationalacademies.org/" target="_self"> Read Our Expert Reports and Published Proceedings </a></li><li><a href="https://www.pnas.org/" target="_self"> Explore PNAS, the Flagship Scientific Journal of NAS </a></li><li><a href="http://pubsindex.trb.org" target="_self"> Access Transportation Research Board Publications </a></li></ul> </div> <button class="accordion">Topics</button> <div class="panel"> <ul><li><a href="https://www.nationalacademies.org/topics/covid-19-resources"> Coronavirus Disease 2019 (COVID-19) </a></li><li><a href="https://www.nationalacademies.org/topics/resources-on-diversity-equity-and-inclusion"> Diversity, Equity, and Inclusion </a></li><li><a href="https://www.nationalacademies.org/topics/climate"> Climate </a></li><li><a href="https://www.nationalacademies.org/topics/economic-recovery"> Economic Recovery </a></li></ul> </div> <button class="no-accordion"><a href="https://www.nationalacademies.org/engagement">Engagement</a></button> <button class="accordion">Opportunities</button> <div class="panel"> <ul><li><a href="https://www.nationalacademies.org/careers"> Careers </a></li><li><a href="https://www.nationalacademies.org/fellowships-and-grants"> Fellowships and Grants </a></li><li><a href="https://www.nationalacademies.org/about/volunteers"> Volunteers </a></li><li><a href="https://www.nationalacademies.org/about/sponsors"> Sponsors </a></li><li><a href="https://www.nationalacademies.org/about/members"> Members </a></li></ul> </div> </div></div> </div> </div> <!--Mobile menu end --> </div> <div class="container flex justify-between"> <div class="global-nav__logo" > <a class="navbar-brand js-scroll-trigger" aria-label="Logo of The National Academies" href="https://www.nationalacademies.org/home"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 315.11 58.33"> <defs><style>.cls-2{fill:#211747;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><polygon class="cls-2" points="3.73 7.55 20.84 25.49 20.84 1.5 17.81 1.5 17.81 18.08 0.7 0.14 0.7 24.06 3.73 24.06 3.73 7.55"/><polygon class="cls-2" points="28.49 24.06 31.84 24.06 38.98 7.25 45.92 23.87 45.99 24.06 49.34 24.06 39.05 0 28.49 24.06"/><polygon class="cls-2" points="153.23 24.06 156.58 24.06 163.72 7.25 170.66 23.87 170.73 24.06 174.08 24.06 163.79 0 153.23 24.06"/><polygon class="cls-2" points="59.72 24.06 62.76 24.06 62.76 4.41 69.52 4.41 69.52 1.5 52.96 1.5 52.96 4.41 59.72 4.41 59.72 24.06"/><rect class="cls-2" x="78.85" y="1.5" width="3.03" height="22.55"/><path class="cls-2" d="M115.69,12.8a11.72,11.72,0,0,0-23.44,0,11.72,11.72,0,0,0,23.44,0m-3.09,0A8.63,8.63,0,1,1,104,4a8.7,8.7,0,0,1,8.63,8.76"/><polygon class="cls-2" points="128.29 7.55 145.4 25.49 145.4 1.5 142.37 1.5 142.37 18.08 125.25 0.14 125.25 24.06 128.29 24.06 128.29 7.55"/><polygon class="cls-2" points="184.65 1.5 181.62 1.5 181.62 24.06 193.7 24.06 193.7 21.16 184.65 21.16 184.65 1.5"/><path class="cls-2" d="M37.15,34.08A11.48,11.48,0,0,0,25.37,45.8,11.36,11.36,0,0,0,37,57.38a11.77,11.77,0,0,0,6.79-2.19l.13-.09V51.2l-.51.51A9.77,9.77,0,0,1,41,53.5a8.9,8.9,0,0,1-3.95,1A8.75,8.75,0,0,1,37,37a9.09,9.09,0,0,1,3.66.82,10.71,10.71,0,0,1,2.78,2l.51.5V36.4l-.12-.09a10.62,10.62,0,0,0-6.67-2.23"/><path class="cls-2" d="M91.91,37.08a12.19,12.19,0,0,0-8.29-2.64H78.85V57h4.74c3.72,0,6-.73,8.22-2.61a10.75,10.75,0,0,0,4-8.65,11,11,0,0,0-3.86-8.65m-2.09,15.1a9.11,9.11,0,0,1-6.1,1.91H81.88V37.34h1.84c5.44,0,9,3.29,9,8.39a8.09,8.09,0,0,1-2.85,6.45"/><polygon class="cls-2" points="104.99 56.99 118.4 56.99 118.4 54.09 108.02 54.09 108.02 46.16 117.11 46.16 117.11 43.26 108.02 43.26 108.02 37.34 118.4 37.34 118.4 34.44 104.99 34.44 104.99 56.99"/><polygon class="cls-2" points="176.17 56.99 189.58 56.99 189.58 54.09 179.2 54.09 179.2 46.16 188.28 46.16 188.28 43.26 179.2 43.26 179.2 37.34 189.58 37.34 189.58 34.44 176.17 34.44 176.17 56.99"/><polygon class="cls-2" points="138.88 51.34 130.31 32.49 126.44 56.99 129.55 56.99 131.62 42.47 138.84 58.33 146.15 42.47 148.13 56.99 151.23 56.99 147.61 32.45 138.88 51.34"/><rect class="cls-2" x="161.31" y="34.44" width="3.03" height="22.55"/><path class="cls-2" d="M209,46.15A10.78,10.78,0,0,0,205.49,44l-2.12-.92a6.8,6.8,0,0,1-2.21-1.33,2,2,0,0,1-.73-1.69A3.07,3.07,0,0,1,203.82,37a3.62,3.62,0,0,1,2.28.69,4.87,4.87,0,0,1,1.2,1.55l.15.3,2.45-1.62-.12-.24a7,7,0,0,0-2.24-2.55,6.34,6.34,0,0,0-3.69-1c-3.8,0-6.45,2.45-6.45,6a4.86,4.86,0,0,0,1.6,3.8,9.51,9.51,0,0,0,2.84,1.78l2.15,1a9.5,9.5,0,0,1,2.56,1.46,3,3,0,0,1,1,2.35,4.2,4.2,0,0,1-7.06,2.92,4.93,4.93,0,0,1-1.24-2.64l-.05-.33-3,.82.06.28a8.32,8.32,0,0,0,2.13,4,6.74,6.74,0,0,0,5,1.87,6.91,6.91,0,0,0,7.24-7A5.58,5.58,0,0,0,209,46.15"/><polygon class="cls-2" points="0 56.99 3.35 56.99 10.49 40.18 17.43 56.81 17.5 56.99 20.86 56.99 10.56 32.93 0 56.99"/><polygon class="cls-2" points="50.21 56.99 53.57 56.99 60.71 40.18 67.64 56.81 67.72 56.99 71.07 56.99 60.78 32.93 50.21 56.99"/><path class="cls-2" d="M233.74,2.1a3.34,3.34,0,0,0-.93-.11,2.53,2.53,0,0,0-1.52.45,1.48,1.48,0,0,0-.63,1.28,2,2,0,0,0,.54,1.44,7.94,7.94,0,0,0,1.64,1.26,13.5,13.5,0,0,1,1.26.89,3.76,3.76,0,0,1,.83,1,2.62,2.62,0,0,1,.33,1.33,3.06,3.06,0,0,1-.5,1.74,3.26,3.26,0,0,1-1.4,1.18,5.06,5.06,0,0,1-2.09.41,8.3,8.3,0,0,1-1.94-.22,3.51,3.51,0,0,1-1.24-.46l.34-2.18h.92l.09,1.62a2.53,2.53,0,0,0,.81.33,3.86,3.86,0,0,0,1,.13,2.93,2.93,0,0,0,1.77-.53,1.68,1.68,0,0,0,.73-1.47,1.91,1.91,0,0,0-.49-1.34,6.42,6.42,0,0,0-1.49-1.16c-.52-.34-1-.65-1.32-.94a3.76,3.76,0,0,1-.9-1,2.64,2.64,0,0,1-.37-1.4,2.64,2.64,0,0,1,.5-1.62,3.23,3.23,0,0,1,1.35-1.08,4.58,4.58,0,0,1,1.86-.37,7,7,0,0,1,1.72.18,3.31,3.31,0,0,1,1.09.45l-.35,2h-.83l-.12-1.47a2.39,2.39,0,0,0-.68-.3"/><path class="cls-2" d="M241,5a2,2,0,0,0-1.35.54,3.66,3.66,0,0,0-.92,1.52,7,7,0,0,0-.33,2.24,3.52,3.52,0,0,0,.43,1.95,1.39,1.39,0,0,0,1.24.66,2.38,2.38,0,0,0,1.1-.24,5.05,5.05,0,0,0,1-.72l.3.57a3.84,3.84,0,0,1-1.14.95,3.43,3.43,0,0,1-1.76.43,2.44,2.44,0,0,1-2-.91,4,4,0,0,1-.73-2.54A6.43,6.43,0,0,1,237.34,7a5.3,5.3,0,0,1,1.55-2.08,3.74,3.74,0,0,1,2.42-.83,5,5,0,0,1,.93.09,3.09,3.09,0,0,1,.81.26l-.42,1.49-.26-.09A2.53,2.53,0,0,0,241,5"/><path class="cls-2" d="M246.1,11.61a.56.56,0,0,0,.07.29.21.21,0,0,0,.18.1.65.65,0,0,0,.34-.11,2.64,2.64,0,0,0,.44-.35l.2.37a2.88,2.88,0,0,1-.68.67,1.71,1.71,0,0,1-1.08.35,1.14,1.14,0,0,1-.78-.23.8.8,0,0,1-.22-.72c0-.25.15-1.25.4-3,.14-.93.31-2.16.52-3.7l-1-.35.09-.52,2.18-.36.34.17-1,7.22ZM246,2.15a.84.84,0,0,1-.23-.59,1.13,1.13,0,0,1,.3-.79.94.94,0,0,1,.71-.32.79.79,0,0,1,.61.24.77.77,0,0,1,.23.57,1.15,1.15,0,0,1-.29.81,1,1,0,0,1-.77.33.75.75,0,0,1-.56-.25"/><path class="cls-2" d="M254,12.43a3.75,3.75,0,0,1-1.95.52,3,3,0,0,1-1.72-.47,2.74,2.74,0,0,1-1-1.23A4,4,0,0,1,249,9.62a6.35,6.35,0,0,1,.59-2.75,5.08,5.08,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.27,2.27,0,0,1,1.64.52A1.79,1.79,0,0,1,255.72,6a2.34,2.34,0,0,1-.89,1.84,5.79,5.79,0,0,1-2.08,1.12,8.1,8.1,0,0,1-2.15.4,3.42,3.42,0,0,0,.37,1.87,1.48,1.48,0,0,0,1.4.79,3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57A4,4,0,0,1,254,12.43m-2.11-7A4.1,4.1,0,0,0,251,6.88a5.9,5.9,0,0,0-.39,1.69,5.31,5.31,0,0,0,1.62-.33,4,4,0,0,0,1.41-.87,1.76,1.76,0,0,0,.57-1.28,1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/><path class="cls-2" d="M261.24,4.62A3.14,3.14,0,0,1,263,4.06a1.67,1.67,0,0,1,1.32.52,2.61,2.61,0,0,1,.46,1.71,15.44,15.44,0,0,1-.34,2.34l-.06.36-.13.74a11.51,11.51,0,0,0-.24,1.85c0,.28.06.42.2.42a1.22,1.22,0,0,0,.72-.39l.18.4a1.54,1.54,0,0,1-.32.36,2.92,2.92,0,0,1-.61.39,1.73,1.73,0,0,1-.79.17.84.84,0,0,1-1-.93,12.88,12.88,0,0,1,.27-1.88l.14-.73.06-.37c.1-.55.19-1,.25-1.45a8.17,8.17,0,0,0,.09-1.16,1.67,1.67,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.33,2.33,0,0,0-1.37.61,5.75,5.75,0,0,0-1.3,1.47l-.75,5.54h-1.46l1-7.43L257.62,5l.12-.58,1.88-.34.32.17-.2,1.84a5.72,5.72,0,0,1,1.5-1.45"/><path class="cls-2" d="M270.77,5a2,2,0,0,0-1.35.54,3.66,3.66,0,0,0-.92,1.52,7,7,0,0,0-.33,2.24,3.52,3.52,0,0,0,.43,1.95,1.39,1.39,0,0,0,1.24.66,2.38,2.38,0,0,0,1.1-.24,5.05,5.05,0,0,0,1-.72l.3.57a3.84,3.84,0,0,1-1.14.95,3.43,3.43,0,0,1-1.76.43,2.44,2.44,0,0,1-2-.91,4,4,0,0,1-.73-2.54A6.43,6.43,0,0,1,267.13,7a5.3,5.3,0,0,1,1.55-2.08,3.76,3.76,0,0,1,2.42-.83,5,5,0,0,1,.93.09,3,3,0,0,1,.81.26l-.42,1.49-.26-.09A2.53,2.53,0,0,0,270.77,5"/><path class="cls-2" d="M278.66,12.43a3.75,3.75,0,0,1-2,.52,3,3,0,0,1-1.71-.47,2.7,2.7,0,0,1-1-1.23,4.19,4.19,0,0,1-.32-1.63,6.35,6.35,0,0,1,.59-2.75,5.17,5.17,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.52A1.82,1.82,0,0,1,280.38,6a2.34,2.34,0,0,1-.89,1.84,5.79,5.79,0,0,1-2.08,1.12,8,8,0,0,1-2.15.4,3.51,3.51,0,0,0,.37,1.87A1.48,1.48,0,0,0,277,12a3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.27,1.11m-2.11-7a4.25,4.25,0,0,0-.87,1.45,6.25,6.25,0,0,0-.39,1.69,5.44,5.44,0,0,0,1.63-.33,4.05,4.05,0,0,0,1.4-.87,1.76,1.76,0,0,0,.57-1.28,1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/><path class="cls-2" d="M286.18,5.13a2.75,2.75,0,0,0-1.09-.21,1.85,1.85,0,0,0-1.13.31,1,1,0,0,0-.43.89,1.43,1.43,0,0,0,.34.93,6.08,6.08,0,0,0,1,.88,10.43,10.43,0,0,1,.88.74,3.1,3.1,0,0,1,.6.76,2.15,2.15,0,0,1,.26,1.05,2.09,2.09,0,0,1-.9,1.81,3.71,3.71,0,0,1-2.22.64,6.32,6.32,0,0,1-1.37-.15,2.1,2.1,0,0,1-.9-.34l.24-1.06a3.41,3.41,0,0,0,.88.44,3.37,3.37,0,0,0,1.18.21,2.21,2.21,0,0,0,1.15-.29,1,1,0,0,0,.48-.92,1.37,1.37,0,0,0-.38-1,6.83,6.83,0,0,0-1.1-.89,3.07,3.07,0,0,1-1.58-2.5,2.16,2.16,0,0,1,.4-1.27,2.64,2.64,0,0,1,1.09-.84,3.81,3.81,0,0,1,1.54-.3,6.32,6.32,0,0,1,1.13.1,4.07,4.07,0,0,1,.8.19l-.26,1.17a1.55,1.55,0,0,0-.6-.39"/><polygon class="cls-2" points="229.34 23.43 236.61 23.43 236.51 25.87 235.6 25.87 235.35 24.25 231.87 24.13 231.29 28.48 234.85 28.36 234.74 29.51 231.21 29.38 230.58 34.05 234.91 33.87 235.77 32.01 236.63 32.18 236.01 34.84 227.85 34.84 227.91 34.24 229.06 34.05 230.29 24.25 229.24 24.09 229.34 23.43"/><path class="cls-2" d="M242.18,26.67a3.16,3.16,0,0,1,1.73-.56,1.67,1.67,0,0,1,1.32.53,2.51,2.51,0,0,1,.46,1.7,14.42,14.42,0,0,1-.34,2.35l-.06.35-.12.74a10.81,10.81,0,0,0-.25,1.86c0,.27.07.41.2.41a1.15,1.15,0,0,0,.72-.38l.19.4a1.54,1.54,0,0,1-.32.35,2.46,2.46,0,0,1-.61.39,1.76,1.76,0,0,1-.79.18c-.65,0-1-.31-1-.94a11.94,11.94,0,0,1,.28-1.87l.14-.74.06-.37c.1-.55.18-1,.24-1.45a8.14,8.14,0,0,0,.1-1.16,1.64,1.64,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.3,2.3,0,0,0-1.38.61,5.67,5.67,0,0,0-1.29,1.46l-.76,5.55h-1.45l1-7.44-.76-.36.13-.59,1.87-.34.32.17-.2,1.85a5.77,5.77,0,0,1,1.5-1.46"/><path class="cls-2" d="M252.77,37.29a4,4,0,0,1-1.55,1.05,5.36,5.36,0,0,1-1.78.32,5.69,5.69,0,0,1-1-.09,3.45,3.45,0,0,1-.76-.19l.32-1.24a5,5,0,0,0,.9.34,3.56,3.56,0,0,0,1.06.17,2.22,2.22,0,0,0,1.45-.39,2.12,2.12,0,0,0,.63-1.41l.29-2a3.73,3.73,0,0,1-1,.84A2.61,2.61,0,0,1,250,35a2.16,2.16,0,0,1-1.89-.88,3.9,3.9,0,0,1-.64-2.35,6.52,6.52,0,0,1,.58-2.6A5.62,5.62,0,0,1,249.73,27a4.15,4.15,0,0,1,2.61-.87,4.47,4.47,0,0,1,.87.1,5.15,5.15,0,0,1,.8.24l1-.34-1.38,9.31a3.37,3.37,0,0,1-.81,1.87m-.61-10.36a2.36,2.36,0,0,0-1.75.74,4.82,4.82,0,0,0-1.07,1.85,7.16,7.16,0,0,0-.35,2.15,3.3,3.3,0,0,0,.37,1.72,1.16,1.16,0,0,0,1,.62,2.09,2.09,0,0,0,1.2-.39,3.12,3.12,0,0,0,.9-.87l.82-5.5a1.85,1.85,0,0,0-1.15-.32"/><path class="cls-2" d="M258.39,33.67a.45.45,0,0,0,.07.28.19.19,0,0,0,.17.1.6.6,0,0,0,.35-.11,3.13,3.13,0,0,0,.44-.35l.2.37a3.17,3.17,0,0,1-.69.68,1.77,1.77,0,0,1-1.08.35,1.14,1.14,0,0,1-.78-.24.82.82,0,0,1-.22-.71c0-.26.16-1.25.4-3,.15-.93.32-2.17.52-3.7l-.95-.36.09-.52,2.19-.35.33.17-1,7.21Zm-.07-9.46a.81.81,0,0,1-.24-.6,1.09,1.09,0,0,1,.3-.78.94.94,0,0,1,.72-.33.81.81,0,0,1,.61.24.82.82,0,0,1,.23.58,1.17,1.17,0,0,1-.29.8,1,1,0,0,1-.77.33.76.76,0,0,1-.56-.24"/><path class="cls-2" d="M265.74,26.67a3.11,3.11,0,0,1,1.72-.56,1.65,1.65,0,0,1,1.32.53,2.5,2.5,0,0,1,.47,1.7,15.68,15.68,0,0,1-.34,2.35l-.06.35-.13.74a11.6,11.6,0,0,0-.24,1.86c0,.27.06.41.2.41a1.2,1.2,0,0,0,.72-.38l.18.4a1.54,1.54,0,0,1-.32.35,2.28,2.28,0,0,1-.61.39,1.73,1.73,0,0,1-.79.18.84.84,0,0,1-1-.94,12.8,12.8,0,0,1,.27-1.87l.14-.74.06-.37c.1-.55.19-1,.25-1.45a8.07,8.07,0,0,0,.09-1.16,1.64,1.64,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.27,2.27,0,0,0-1.37.61,5.7,5.7,0,0,0-1.3,1.46l-.75,5.55h-1.46l1-7.44-.75-.36.12-.59,1.88-.34.32.17-.2,1.85a5.77,5.77,0,0,1,1.5-1.46"/><path class="cls-2" d="M276.11,34.48a3.7,3.7,0,0,1-1.95.52,3,3,0,0,1-1.71-.46,2.87,2.87,0,0,1-1-1.23,4.24,4.24,0,0,1-.32-1.64,6.53,6.53,0,0,1,.59-2.75,5.17,5.17,0,0,1,1.63-2,3.76,3.76,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.53,1.76,1.76,0,0,1,.55,1.36,2.31,2.31,0,0,1-.88,1.84A5.83,5.83,0,0,1,274.87,31a9.08,9.08,0,0,1-2.15.4,3.49,3.49,0,0,0,.36,1.86,1.48,1.48,0,0,0,1.4.79,3.16,3.16,0,0,0,1.35-.31,3.9,3.9,0,0,0,1.22-.9l.33.57a3.93,3.93,0,0,1-1.27,1.1m-2.1-7a4.12,4.12,0,0,0-.88,1.44,6.37,6.37,0,0,0-.38,1.7,5.17,5.17,0,0,0,1.62-.34,3.78,3.78,0,0,0,1.4-.87,1.74,1.74,0,0,0,.57-1.28,1.31,1.31,0,0,0-.28-.91,1,1,0,0,0-.78-.32,1.68,1.68,0,0,0-1.27.58"/><path class="cls-2" d="M284,34.48a3.7,3.7,0,0,1-2,.52,3,3,0,0,1-1.71-.46,2.87,2.87,0,0,1-1-1.23,4.24,4.24,0,0,1-.32-1.64,6.53,6.53,0,0,1,.59-2.75,5.26,5.26,0,0,1,1.63-2,3.79,3.79,0,0,1,2.3-.77,2.31,2.31,0,0,1,1.64.53,1.76,1.76,0,0,1,.55,1.36,2.31,2.31,0,0,1-.88,1.84A5.74,5.74,0,0,1,282.73,31a9.08,9.08,0,0,1-2.15.4,3.4,3.4,0,0,0,.37,1.86,1.45,1.45,0,0,0,1.39.79,3.2,3.2,0,0,0,1.36-.31,3.86,3.86,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.28,1.1m-2.1-7a4.12,4.12,0,0,0-.88,1.44,6.37,6.37,0,0,0-.38,1.7,5.17,5.17,0,0,0,1.62-.34,3.7,3.7,0,0,0,1.4-.87,1.74,1.74,0,0,0,.57-1.28,1.31,1.31,0,0,0-.28-.91,1,1,0,0,0-.78-.32,1.7,1.7,0,0,0-1.27.58"/><path class="cls-2" d="M290.05,27.3a3.77,3.77,0,0,1,1-.83,2.34,2.34,0,0,1,1.22-.34.83.83,0,0,1,.5.11l-.33,1.62a1.45,1.45,0,0,0-.75-.15,2.3,2.3,0,0,0-1.29.44,2.89,2.89,0,0,0-1,1.22l-.7,5.47h-1.48l1-7.41-.77-.38.11-.57,1.81-.35.31.18-.11,1.37-.08.61a4.09,4.09,0,0,1,.6-1"/><path class="cls-2" d="M295.29,33.67a.45.45,0,0,0,.07.28.19.19,0,0,0,.17.1.6.6,0,0,0,.35-.11,3.13,3.13,0,0,0,.44-.35l.2.37a3.17,3.17,0,0,1-.69.68,1.77,1.77,0,0,1-1.08.35,1.14,1.14,0,0,1-.78-.24.82.82,0,0,1-.22-.71c0-.26.16-1.25.4-3,.15-.93.32-2.17.52-3.7l-.95-.36.09-.52,2.19-.35.33.17-1,7.21Zm-.07-9.46a.81.81,0,0,1-.24-.6,1.09,1.09,0,0,1,.3-.78.94.94,0,0,1,.72-.33.81.81,0,0,1,.61.24.82.82,0,0,1,.23.58,1.17,1.17,0,0,1-.29.8,1,1,0,0,1-.77.33.76.76,0,0,1-.56-.24"/><path class="cls-2" d="M302.33,26.67a3.14,3.14,0,0,1,1.73-.56,1.67,1.67,0,0,1,1.32.53,2.56,2.56,0,0,1,.46,1.7,15.68,15.68,0,0,1-.34,2.35l-.06.35-.12.74a10.81,10.81,0,0,0-.25,1.86c0,.27.07.41.2.41a1.2,1.2,0,0,0,.72-.38l.19.4a1.64,1.64,0,0,1-.33.35,2.22,2.22,0,0,1-.6.39,1.76,1.76,0,0,1-.79.18c-.65,0-1-.31-1-.94a12.8,12.8,0,0,1,.27-1.87l.14-.74.06-.37c.11-.55.19-1,.25-1.45a8.07,8.07,0,0,0,.09-1.16,1.73,1.73,0,0,0-.2-1,.88.88,0,0,0-.73-.27,2.3,2.3,0,0,0-1.38.61,5.7,5.7,0,0,0-1.3,1.46l-.75,5.55h-1.46l1-7.44-.75-.36.13-.59,1.87-.34.32.17-.2,1.85a5.77,5.77,0,0,1,1.5-1.46"/><path class="cls-2" d="M312.92,37.29a4,4,0,0,1-1.55,1.05,5.36,5.36,0,0,1-1.78.32,5.69,5.69,0,0,1-1-.09,3.57,3.57,0,0,1-.77-.19l.33-1.24a5,5,0,0,0,.9.34,3.51,3.51,0,0,0,1.06.17,2.22,2.22,0,0,0,1.45-.39,2.11,2.11,0,0,0,.62-1.41l.3-2a3.73,3.73,0,0,1-1,.84,2.63,2.63,0,0,1-1.33.34,2.16,2.16,0,0,1-1.89-.88,3.9,3.9,0,0,1-.64-2.35,6.35,6.35,0,0,1,.58-2.6A5.62,5.62,0,0,1,309.88,27a4.11,4.11,0,0,1,2.61-.87,4.56,4.56,0,0,1,.87.1,5.15,5.15,0,0,1,.8.24l.95-.34-1.38,9.31a3.37,3.37,0,0,1-.81,1.87m-.62-10.36a2.35,2.35,0,0,0-1.74.74,4.7,4.7,0,0,0-1.07,1.85,7.16,7.16,0,0,0-.35,2.15,3.3,3.3,0,0,0,.37,1.72,1.14,1.14,0,0,0,1,.62,2,2,0,0,0,1.19-.39,3.3,3.3,0,0,0,.91-.87l.82-5.5a1.88,1.88,0,0,0-1.16-.32"/><path class="cls-2" d="M237.11,49.24c.83-1.82,1.39-3.07,1.65-3.76h3.29l-.06.66-1.37.17-.29,9.8,1.31.18-.07.6h-4.22l.06-.6,1.4-.18.35-6.79.2-3.12L238.2,49l-3.57,8-.83.11-1.26-7.89-.47-2.92-.63,3-1.64,6.91,1.55.18-.06.6h-4l.06-.6,1.28-.18,2.47-9.8-1.41-.17.09-.66h3.56L234.56,53l.2,1.57.67-1.53c.28-.69.84-1.95,1.68-3.77"/><path class="cls-2" d="M247.75,56.54a3.75,3.75,0,0,1-1.95.52,3,3,0,0,1-1.71-.47,2.7,2.7,0,0,1-1-1.23,4.19,4.19,0,0,1-.32-1.63,6.35,6.35,0,0,1,.59-2.75,5.17,5.17,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.52,1.82,1.82,0,0,1,.55,1.37,2.34,2.34,0,0,1-.89,1.84A5.79,5.79,0,0,1,246.5,53a8,8,0,0,1-2.15.4,3.51,3.51,0,0,0,.37,1.87,1.48,1.48,0,0,0,1.4.79,3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.27,1.11m-2.11-7a4.25,4.25,0,0,0-.87,1.45,6.25,6.25,0,0,0-.39,1.69,5.44,5.44,0,0,0,1.63-.33,4.05,4.05,0,0,0,1.4-.87A1.76,1.76,0,0,0,248,50.2a1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/><path class="cls-2" d="M257.17,55.72q0,.39.24.39a1.19,1.19,0,0,0,.7-.39l.22.39a3.42,3.42,0,0,1-.67.62,1.68,1.68,0,0,1-1,.31c-.6,0-.9-.32-.9-1v-.34a3.94,3.94,0,0,1-1.08.94,2.72,2.72,0,0,1-1.44.38,2.13,2.13,0,0,1-1.87-.88,4,4,0,0,1-.63-2.34,6.41,6.41,0,0,1,.58-2.62A5.56,5.56,0,0,1,253,49a4.1,4.1,0,0,1,2.61-.86,4,4,0,0,1,1.15.18l.41-2.76-1-.26.13-.54,2.16-.28.34.2-1.6,10.84a.79.79,0,0,0,0,.17M255.42,49a2.32,2.32,0,0,0-1.74.75,4.64,4.64,0,0,0-1.06,1.84,7.15,7.15,0,0,0-.34,2.16,3.44,3.44,0,0,0,.36,1.72,1.12,1.12,0,0,0,1,.61,2.06,2.06,0,0,0,1.23-.41,3.11,3.11,0,0,0,.9-.91l.83-5.42a1.94,1.94,0,0,0-1.2-.34"/><path class="cls-2" d="M262.14,55.72a.57.57,0,0,0,.06.29.24.24,0,0,0,.18.1.71.71,0,0,0,.35-.11,3.84,3.84,0,0,0,.44-.35l.19.37a2.88,2.88,0,0,1-.68.67,1.71,1.71,0,0,1-1.08.35,1.19,1.19,0,0,1-.78-.23.8.8,0,0,1-.22-.72c0-.25.15-1.25.4-3,.14-.93.32-2.16.52-3.7l-.95-.35.09-.52,2.18-.36.34.17-1,7.22Zm-.07-9.46a.81.81,0,0,1-.24-.6,1.12,1.12,0,0,1,.3-.78,1,1,0,0,1,.71-.32.83.83,0,0,1,.62.24.81.81,0,0,1,.23.57,1.2,1.2,0,0,1-.29.81,1,1,0,0,1-.77.33.73.73,0,0,1-.56-.25"/><path class="cls-2" d="M269.42,49.14a2,2,0,0,0-1.35.54,3.6,3.6,0,0,0-.92,1.52,7,7,0,0,0-.33,2.24,3.52,3.52,0,0,0,.43,2,1.39,1.39,0,0,0,1.24.66,2.38,2.38,0,0,0,1.1-.24,5.05,5.05,0,0,0,1-.72l.3.57a4,4,0,0,1-1.14,1A3.43,3.43,0,0,1,268,57a2.44,2.44,0,0,1-2-.91,4,4,0,0,1-.73-2.54,6.43,6.43,0,0,1,.53-2.51A5.3,5.3,0,0,1,267.33,49a3.69,3.69,0,0,1,2.42-.83,5,5,0,0,1,.93.09,3.09,3.09,0,0,1,.81.26L271.07,50l-.26-.09a2.53,2.53,0,0,0-1.39-.78"/><path class="cls-2" d="M274.77,55.72a.56.56,0,0,0,.07.29.24.24,0,0,0,.18.1.71.71,0,0,0,.35-.11,3.15,3.15,0,0,0,.43-.35l.2.37a2.88,2.88,0,0,1-.68.67,1.71,1.71,0,0,1-1.08.35,1.19,1.19,0,0,1-.78-.23.8.8,0,0,1-.22-.72c0-.25.15-1.25.4-3,.14-.93.32-2.16.52-3.7l-1-.35.09-.52,2.18-.36.34.17-1,7.22Zm-.07-9.46a.85.85,0,0,1-.23-.6,1.12,1.12,0,0,1,.3-.78.94.94,0,0,1,.71-.32.79.79,0,0,1,.61.24.78.78,0,0,1,.24.57,1.16,1.16,0,0,1-.3.81,1,1,0,0,1-.76.33.75.75,0,0,1-.57-.25"/><path class="cls-2" d="M282,48.73a3.09,3.09,0,0,1,1.73-.56,1.65,1.65,0,0,1,1.32.52,2.54,2.54,0,0,1,.46,1.7,15.09,15.09,0,0,1-.33,2.35l-.07.36-.12.73a11.6,11.6,0,0,0-.24,1.86c0,.28.06.42.19.42a1.22,1.22,0,0,0,.73-.39l.18.4a1.77,1.77,0,0,1-.32.36,3.21,3.21,0,0,1-.61.39,1.86,1.86,0,0,1-.79.17c-.64,0-1-.31-1-.93a13.42,13.42,0,0,1,.28-1.88l.14-.73.06-.37c.1-.56.18-1,.25-1.45a8.17,8.17,0,0,0,.09-1.16,1.67,1.67,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.31,2.31,0,0,0-1.37.61,5.58,5.58,0,0,0-1.3,1.47l-.75,5.54H278.2l1-7.43-.75-.37.12-.58,1.87-.34.33.17-.2,1.84A5.85,5.85,0,0,1,282,48.73"/><path class="cls-2" d="M292.42,56.54a3.75,3.75,0,0,1-1.95.52,3,3,0,0,1-1.71-.47,2.7,2.7,0,0,1-1-1.23,4.19,4.19,0,0,1-.32-1.63A6.35,6.35,0,0,1,288,51a5.17,5.17,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.52,1.82,1.82,0,0,1,.55,1.37,2.34,2.34,0,0,1-.89,1.84A5.79,5.79,0,0,1,291.17,53a8,8,0,0,1-2.15.4,3.51,3.51,0,0,0,.37,1.87,1.48,1.48,0,0,0,1.4.79,3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.27,1.11m-2.11-7a4.25,4.25,0,0,0-.87,1.45,6.25,6.25,0,0,0-.39,1.69,5.44,5.44,0,0,0,1.63-.33,4.05,4.05,0,0,0,1.4-.87,1.76,1.76,0,0,0,.57-1.28,1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/></g></g> </svg> </a> </div> <div class='global-nav-nap-menu' id="header-large"> <div class="dropdown" id="sitelinks-mobile"> <button class="btn btn-secondary dropdown-toggle" type="button" id="dropdownMenuButton" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class="hidden-xxs">Navigate to</span> </button> <div class="dropdown-menu" aria-labelledby="dropdownMenuButton"> <a class="dropdown-item" href="/content/help">Help</a> <a class="dropdown-item" href="/content/help-with-ordering">Ordering Information</a> <a class="dropdown-item" href="/new" >New Releases</a> <a class="dropdown-item" href="/author">Browse by Division</a> <a class="dropdown-item" href="/topic" class="site-links"> Browse by Topic</a> </div> </div> <ul class="user-links"> <li><a href="/"><span class="icon-menu" data-icon=""> </span> NAP Home</a></li> <li class="dropdown nap-mega-menu mega-dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="icon-menu" data-icon=""> </span> Browse <span class="caret"></span></a> <ul class="dropdown-menu mega-dropdown-menu row" id="nap-mega-menu-inner"> <li class="d-flex" > <ul class="browse-col" > <li class="dropdown-header"><a href="/author">Publications by Division</a></li> <li><a href="/author/DBASSE/">Division of Behavioral and Social Sciences and Education</a> </li> <li><a href="/author/DELS/">Division on Earth and Life Studies</a></li> <li><a href="/author/DEPS/">Division on Engineering and Physical Sciences</a> </li> <li><a href="/author/GRP/">Gulf Research Program</a></li> <li><a href="/author/HMD/">Health and Medicine Division</a></li> <li><a href="/author/PGA/">Policy and Global Affairs</a></li> <li><a href="/author/TRB/">Transportation Research Board</a></li> <li><hr style="margin-left: 21px; margin-right: 20px;"></li> <li><a href="/author/NAS">National Academy of Sciences</a></li> <li><a href="/author/NAE">National Academy of Engineering</a></li> <li><a href="/author/NAM">National Academy of Medicine</a></li> </ul> <ul class="browse-col" > <li class="dropdown-header"><a href="/topic">Publications by Topic</a></li> <li><a href="/topic/276">Agriculture</a></li> <li><a href="/topic/277">Behavioral and Social Sciences</a></li> <li><a href="/topic/292">Biography and Autobiography</a></li> <li><a href="/topic/278">Biology and Life Sciences</a></li> <li><a href="/topic/279">Computers and Information Technology</a></li> <li><a href="/topic/280">Conflict and Security Issues</a></li> <li><a href="/topic/281">Earth Sciences</a></li> <li><a href="/topic/282">Education</a></li> <li><a href="/topic/283">Energy and Energy Conservation</a></li> <li><a href="/topic/284">Engineering and Technology</a></li> </ul> <ul class="browse-col"> <li><br> </li> <li><a href="/topic/285">Environment and Environmental Studies</a></li> <li><a href="/topic/287">Food and Nutrition</a></li> <li><a href="/topic/288">Health and Medicine</a></li> <li><a href="/topic/289">Industry and Labor</a></li> <li><a href="/topic/290">Math, Chemistry, and Physics</a></li> <li><a href="/topic/423">Policy for Science and Technology</a></li> <li><a href="/topic/293">Space and Aeronautics</a></li> <li><a href="/topic/478">Surveys and Statistics</a></li> <li><a href="/topic/294">Transportation and Infrastructure</a></li> </ul> <ul class="browse-col vertical-divider"> <li class="dropdown-header"><br></li> <li><a href="/collection">Searchable Collections</a></li> <li><a href="/new">New Releases</a></li> </ul> </li> </ul> </li> <li class="dropdown"> <a class="dropdown-toggle" data-toggle="dropdown" href="#"><span class="icon-menu" data-icon=""> </span> MyNAP <span class="caret"></span></a> <ul class="dropdown-menu"> <li><a href="/login.php" class="userOff"><span class="icon-menu" data-icon=""> </span> Login</a></li> <li><a href="/login.php?action=new" class="register"><span class="icon-menu" data-icon=""> </span> Register</a></li> </ul> </li> <li class="naphelp"><a href="/content/help"><span class="icon-menu" data-icon=""> </span> HELP</a></li> <li> <a href="/cart/cart.php"> <span class="icon-menu" data-icon=""> </span> Cart </a> </li> </ul> </div> <div class="global-nav__menus"> <div class="global-nav__menus--upper"> <div class="global-nav__controls"> <div class="controls__social"><!----></div> <button aria-label="Search" class="controls__search" onclick="toggleSearch()"><span class="search__label"> Search </span> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""> <path d="M13.8 21.6a7.8 7.8 0 110-15.6 7.8 7.8 0 010 15.6zm0-2a5.8 5.8 0 100-11.6 5.8 5.8 0 000 11.6z"></path> <path d="M17.899 19.325l1.414-1.414 4.322 4.322-1.414 1.414z"></path> </svg> </button> <button class="controls__menu"> Menu <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""> <path fill-rule="evenodd" d="M8 23h15v-2.5H8V23zm0-6.25h15v-2.5H8v2.5zM8 8v2.5h15V8H8z"></path> </svg> </button> <button class="controls__nav" onclick="toggleMenu(); myMenu(this)"> Global Menu <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="menu-off"> <path fill-rule="evenodd" d="M8 23h15v-2.5H8V23zm0-6.25h15v-2.5H8v2.5zM8 8v2.5h15V8H8z"></path> </svg> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="menu-on"><path fill-rule="evenodd" d="M20.728 8l1.818 1.818-5.455 5.455 5.455 5.455-1.818 1.818-5.455-5.455-5.455 5.455L8 20.728l5.455-5.455L8 9.818 9.818 8l5.455 5.455z"></path></svg> <!----></button> </div> </div> <div class="global-nav__menus--lower" > <nav role="navigation" class="global-nav__links " > <ul role="menu"> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/about" target="_self" role="menuitem" class="global-nav__link"><span> About Us </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"> <ul class="global-nav__links-widget" type="nas-nav-links-widget"> <li><a rel="" title="" href="https://www.nationalacademies.org/about" target="_self"> About Us </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/organization" target="_self"> Our Program Divisions </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/the-national-academies" target="_self"> Our Three Academies </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/leadership" target="_self"> Leadership </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/newsroom" target="_self"> Newsroom </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/ocga" target="_self"> Government Affairs </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/giving" target="_self"> Giving </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/institutional-policies-and-procedures/statement-on-diversity-and-inclusion" target="_self"> Statement on Diversity and Inclusion </a></li> </ul> </div> <div class="dropdown__right"> <p> <p>The National Academies of Sciences, Engineering, and Medicine are private, nonprofit institutions that provide expert advice on some of the most pressing challenges facing the nation and world. Our work helps shape sound policies, inform public opinion, and advance the pursuit of science, engineering, and medicine.</p> </p> <a rel="" title="Learn More" href="https://www.nationalacademies.org/about" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> Learn More <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/events" target="_self" role="menuitem" class="global-nav__link"><span> Events </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"><span class="dropdown__left--default"><img src="https://nasemoceprodcomm-nationalacademies.cec.ocp.oraclecloud.com/content/published/api/v1.1/assets/CONTC88C4C0CA7CE4E7FA8630DEF6E1B3A8F/native/NAS+Building.jpg?cb=_cache_a061&channelToken=d2dc71410b56f1d61406108f1738d0f4" alt=""></span></div> <div class="dropdown__right"> <p> <p>Throughout any given year, the National Academies convene hundreds of conferences, workshops, symposia, forums, roundtables, and other gatherings that attract the finest minds in academia and the public and private sectors. These venues for discussion and debate are essential for allowing the scientific process to unfold.</p> </p> <a rel="" title="See All Events" href="https://www.nationalacademies.org/events" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> See All Events <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/our-work" target="_self" role="menuitem" class="global-nav__link"><span> Our Work </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"> <ul class="global-nav__links-widget" type="nas-nav-links-widget"> <li><a rel="" title="" href="https://www.nationalacademies.org/our-work" target="_self"> Our Work </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/our-study-process" target="_self"> Our Study Process </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/institutional-policies-and-procedures/conflict-of-interest-policies-and-procedures" target="_self"> Conflict of Interest Policies and Procedures </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/institutional-policies-and-procedures/project-comments-and-information" target="_self"> Project Comments and Information </a></li> </ul> </div> <div class="dropdown__right"> <p> <p>The National Academies of Sciences, Engineering, and Medicine are the nation's pre-eminent source of high-quality, objective advice on science, engineering, and health matters. <br> <br> Top experts participate in our projects, activities, and studies to examine and assemble evidence-based findings to address some of society's greatest challenges.</p> </p> <a rel="" title="Learn More" href="https://www.nationalacademies.org/our-work" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> Learn More <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/publications" target="_self" role="menuitem" class="global-nav__link"><span> Publications </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"> <ul class="global-nav__links-widget" loadchildren="true" type="nas-nav-links-widget"> <li><a rel="" title="" href="https://nap.nationalacademies.org/" target="_self"> Read Our Expert Reports and Published Proceedings </a></li> <li><a rel="" title="" href="https://www.pnas.org/" target="_self"> Explore PNAS, the Flagship Scientific Journal of NAS </a></li> <li><a rel="" title="" href="http://pubsindex.trb.org/" target="_self"> Access Transportation Research Board Publications </a></li> </ul> </div> <div class="dropdown__right"> <p>Our peer-reviewed reports present the evidence-based consensus of committees of experts. Published proceedings record the presentations and discussions that take place at hundreds of conferences, workshops, symposia, forums, roundtables, and other gatherings every year. And, our prestigious journals publish the latest scientific findings on a wide range of topics.</p> <a rel="" title="Learn More" href="https://www.nationalacademies.org/publications" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> Learn More <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/topics" target="_self" role="menuitem" class="global-nav__link"><span> Topics </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"> <ul class="global-nav__links-widget" type="nas-nav-links-widget"> <li><a rel="" title="" href="https://www.nationalacademies.org/topics/covid-19-resources" target="_self"> Coronavirus Disease 2019 (COVID-19) </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/topics/resources-on-diversity-equity-and-inclusion" target="_self"> Diversity, Equity, and Inclusion </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/topics/climate" target="_self"> Climate </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/topics/economic-recovery" target="_self"> Economic Recovery </a></li> </ul> </div> <div class="dropdown__right"> <p> <p>Discover what the National Academies are doing in various topic areas to strengthen the fields of science, engineering, and medicine and their capacity to contribute to the well-being of our nation and the world.</p> </p> <a rel="" title="See All Topics" href="https://www.nationalacademies.org/topics" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> See All Topics <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/engagement" target="_self" role="menuitem" class="global-nav__link"><span> Engagement </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"><span class="dropdown__left--default"><img src="https://nasemoceprodcomm-nationalacademies.ocecdn.oraclecloud.com/content/published/api/v1.1/assets/CONTC267CED7812243AB86410D548C73D067/native/engagement-nav.jpg?cb=_cache_a061&channelToken=d2dc71410b56f1d61406108f1738d0f4" alt=""></span></div> <div class="dropdown__right"> <p> <p>A powerful way to foster appreciation for the impact of science and critical and innovative thinking is through art and the humanities. Learn more about the National Academies’ various programs designed to connect, engage, and inspire.</p> </p> <a rel="" title="Learn More" href="https://www.nationalacademies.org/engagement" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> Learn More <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> <li role="none" class="has-dropdown"><a href="https://www.nationalacademies.org/opportunities" target="_self" role="menuitem" class="global-nav__link"><span> Opportunities </span></a> <div class="global-nav__dropdown"> <div class="dropdown__left"> <ul class="global-nav__links-widget" type="nas-nav-links-widget"> <li><a rel="" title="" href="https://www.nationalacademies.org/careers" target="_self"> Careers </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/fellowships-and-grants" target="_self"> Fellowships and Grants </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/volunteers" target="_self"> Volunteers </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/sponsors" target="_self"> Sponsors </a></li> <li><a rel="" title="" href="https://www.nationalacademies.org/about/members" target="_self"> Members </a></li> </ul> </div> <div class="dropdown__right"> <p>Make a real impact on the scientific, engineering, and health-related challenges facing society. Whether as a sponsor or donor, a member or volunteer, or an employee or fellow, you can make a difference.</p> <a rel="" title="Learn More" href="https://www.nationalacademies.org/opportunities" target="_self" role="menuitem" class="button button--secondary button--outline button--icon"> Learn More <span class="button__icon"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class="w2 h2"> <path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path> <path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path> </svg> </span></a></div> </div> </li> </ul> </nav> </div> </div> </div> <!----></header> <a name="content"></a> <script> $(".controls__menu").click(function(){ $("#mobile-menu").toggle(); }); function toggleMenu() { var element = document.getElementById("lowerMenu"); element.classList.toggle("menu-is-hidden"); } function myMenu(x) { x.classList.toggle("invert"); } function toggleSearch() { var element = document.getElementById("lowerMenu"); element.classList.toggle("search-is-hidden"); } </script> <!--accordion--> <script> var acc = document.getElementsByClassName("accordion"); var i; for (i = 0; i < acc.length; i++) { acc[i].addEventListener("click", function() { this.classList.toggle("active"); var panel = this.nextElementSibling; if (panel.style.maxHeight) { panel.style.maxHeight = null; } else { panel.style.maxHeight = panel.scrollHeight + "px"; } }); } </script> <div class="container"> <script language="JavaScript" type="text/javascript"> <!-- // set up formatting variables var phrase_in_context='<font size=1>Key Phrase In Context: '; var p_i_c_table='<table width="97%" bgcolor=darkgray cellpadding=5 cellspacing=1 id=skimresults><tbody>'; var newwindow; function goe(url) { newwindow=window.open(url,name,'height=400,scrollbar=yes,left=100,top=70,resizable=yes,scrollbars=yes,toolbar=yes,status=yes,menubar=yes,location=yes'); if (window.focus) {newwindow.focus()} } var pewwindow; function poe(url) { pewwindow=window.open(url,name,'width=700;scrollbar=yes,left=80,top=90,resizable=yes,scrollbars=yes,toolbar=yes,status=yes,menubar=yes,location=yes'); if (window.focus) {pewwindow.focus()} } var yewwindow; function yoe(url) { yewwindow=window.open(url,name,'height=400,scrollbar=yes,left=120,top=110,resizable=yes,scrollbars=yes,toolbar=yes,status=yes,menubar=yes,location=yes'); if (window.focus) {yewwindow.focus()} } var mewwindow; function moe(url) { mewwindow=window.open(url,name,'height=400,scrollbar=yes,left=140,top=130,resizable=yes,scrollbars=yes,toolbar=yes,status=yes,menubar=yes,location=yes'); if (window.focus) {mewwindow.focus()} } var msewwindow; function msoe(url) { msewwindow=window.open(url,name,'height=400,scrollbar=yes,left=90,top=110,resizable=yes,scrollbars=yes,toolbar=yes,status=yes,menubar=yes,location=yes'); if (window.focus) {msewwindow.focus()} } var sewwindow; function soe(url) { sewwindow=window.open(url,name,'height=400,scrollbar=yes,left=160,top=150,resizable=yes,scrollbars=yes,toolbar=yes,status=yes,menubar=yes,location=yes'); if (window.focus) {sewwindow.focus()} } function show_engines() { document.getElementById("infospace").innerHTML = '<table cellpadding=0 cellspacing=4><tbody><tr><td colspan=2 align=center><div class="optitle">WebResearch Builder ON</div>' + '<font size=1>[<a href="javascript:click_phrases();">Phrase Click</a>] [<a href="javascript:show_start();">Skim View</a>]<br> ' + 'Click on term to display Web Searches<hr size=1 color=darkblue></td></tr>' + '<tr><td valign=top style=\"linkphrase\"><ul></ul></td><td valign=top style=\"linkphrase\"><ul></ul></td></tr>' + '</tbody></table>'; document.getElementById("text2").innerHTML = '<font size=2><br><b>Special WebResearch Feature</b><br><br>Click on the key terms to the left (extracted from this report) to display lists of two-phrase, ' + 'targetted searches to send to Google for Web-wide searches, or to the National Academies Press Discovery Engine for searching more than 3300 publications from the National Academies.<br>' + '<br>This allows one-click searches on combination terms like <br><br><b>"" and ""</b></font>' + '<br>NOTE: Clicking on Google or National Academies Press will open a new window displaying search results.<br><br></font>' + '<hr size=1 color=darkblue>Search for <font size=2><b></b></font> ...</b><hr size=1 color=darkblue>' + '<table bgcolor=white width=100%><tbody>' + '</tbody></table><hr color=darkblue size=1>'; } function click_phrases() { document.getElementById("infospace").innerHTML = '<table cellpadding=0 cellspacing=4><tbody><tr><td colspan=2 align=center><div class=\"optitle\">Phrase Click ON</div><font size=1>[<a href=\"#top\" onclick=\"rollover_phrases()\">Phrase Rollover' + '</a>] [<a href=\"javascript:DoDisplay(\'daf_test\')\;\">WebResearch Builder</a>] [<a href=\"javascript:show_start()\;\">Skim View</a>]<br>&bull\; key term (#pages)' + '</font><hr size=1 color=darkblue></td></tr><tr><td valign=top><ul></ul></td><td valign=top><ul></ul></td></tr></tbody></table>'; } function show_start() { // should return to the "chapter skim"; // html-preps top N number to have in infospace var displayit=''; for (x=0;x<TopPhrases.length;x++){ // for (x=0;x<30;x++){ if (x==5|x==10|x==15|x==20|x==25|x==30|x==35|x==40|x==45|x==50|x==55|x==60|x==65|x==70){ displayit=displayit + '<div class=ider><br></div>'; } var thisphrase=TopPhrases[x]; var passphrase=thisphrase.replace('_','+'); var showphrase=thisphrase.replace('_',' '); displayit= displayit + '<li class="xy"><a href="javascript:x_' + thisphrase + '();">' + showphrase + '</a></li>'; } document.getElementById("text2").innerHTML = '<table width=\"97%\" bgcolor=darkgray cellpadding=5 cellspacing=1 id=\"skimresults\"><tbody><tr bgcolor=white><td valign=top><div class=fromblock>From ' + 'page 79...</div><div class=foundblock>... <font class=\"btext\">will affect a significantly larger part of the DAF than seems obvious at first glance, ' + 'as the committee expects AI to be embedded throughout the entire DAF over the next decade. This chapter discusses the actual scope of the impact of these ' + 'advancements -- not only on the test community but also on the requirements processes and DAF culture.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=79\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... <font class=\"btext\">Space ' + 'Force Operational Test Agency (OTA) , the DAF CDAO, and operational air, intelligence, C2, space, and cyber units.1 This process should in clude assessing ' + 'what broader DAF-wide organizational and governance changes are needed to reflect the differences between AI T&E and T&E for all other Air Force systems ' + 'and capabilities.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 81...</div><div class=foundblock>... <font class=\"btext\">also ' + 'recently established a digital transformation office (DTO) within AFMC; the AFMC commander used this similar approach to rectify the shortfall in implementing ' + 'an effective digital strategy in the Air Force.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=81\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 82...</div><div class=foundblock>... <font class=\"btext\">At ' + 'the same time, while overall model performance is expected to improve over time as more operational data are ingested, absorption of more data could also ' + 'lead to significant reductions in performance if the new data are corrupted or poisoned or the AI model is subject to other forms of adversarial attacks ' + '(see Chapter 5)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=82\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... <font class=\"btext\">3. ' + 'It also notes that \"Programs executing the software acquisition pathway will not be treated as major defense acquisition programs,\" p.</font></div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... <font class=\"btext\">Still, all short-term solutions ' + 'will continue to evolve over time through an itera tive, interactive process as air force end-users and personnel within responsible test organizations ' + 'gain more experience with writing AI-centric T&E require ments and with AI T&E processes and practices and as AI T&E becomes more automated and test results ' + 'become more explainable. The committee echoes the NSCAI\'s recommendation to the military services to \"establish a TEVV framework and culture that integrates ' + 'testing as a continuous part of requirements specifica tion, development, deployment, training, and maintenance and includes run-time monitoring of operational ' + 'behavior.\"8 Section 255 of the FY2020 National Defense Authorization Act (NDAA)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 85...</div><div class=foundblock>... <font class=\"btext\"> 11 ' + 'DoDI 5000.89 requires a test strategy when using the software acquisition pathway, and notes that this pathway \"focuses on modern iterative software development ' + 'techniques such as agile, lean, and development security operations, which promise faster delivery of working code to the user. The goal of this software ' + 'acquisition pathway is to achieve continuous integration and continuous delivery to the maximum extent possible\" (p.</font></div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=85\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... <font class=\"btext\">Members of Project Maven \"translated\" ' + 'esoteric T&E metrics into terms that were most relevant to operational end-users. Because formal requirements had not been established for AI model performance, ' + 'once the Maven team had completed data quality assurance, T&E on each model, integration testing in the Maven Integration Lab, and live-fly testing, user ' + 'acceptance of each trained model and follow-on updates to those fielded models, was based primarily on an agreement between the Maven team and operational ' + 'users that the models had demonstrated adequate performance under operational conditions (as compared to the baseline performance achieved with existing, ' + 'non-AI systems)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... <font class=\"btext\">, ' + 'and DAF CDAO, should be to establish \"a process for writing testable and verifiable AI requirement specifications that characterize realistic operational ' + 'performance,\" and to provide \"testing methodologies and metrics that enable evaluation of these requirements -- including principles of ethical and responsible ' + 'AI, trustworthiness, robustness, and adversarial resilience.\"15 As noted above, the iterative and interactive dialogue between end-users, testers, and ' + 'the broader AI community will help operators and testers agree on request for proposal/request for information (RFP/RFI) and contract language, help endusers ' + 'understand how AI performance will be assessed by testers, and help testers develop appropriate test metrics and performance measures.</font></div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... <font class=\"btext\">When the committee refers to requirements, ' + 'it includes the need for DAF wide investments in enabling capabilities that support AI T&E at enterprise scale. Because of AI\'s unique characteristics ' + 'and the uncertainties associated with AI performance in operational environments, the committee recommends that the DAF prioritize investments for digital ' + 'modernization of the DAF test enterprise and for implementing an enterprise-level T&E architecture, enabled to the maximum extent possible by the OSD CDAO,16 ' + 'OSD DOT&E, OSD DASD(DT&E)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... <font class=\"btext\">and ' + 'PEOs, should provide standardized contract options to address the need for TQD and machine-readable data, along with options for intellectual property ' + '(IP) protections and ownership of data rights and licenses for both commercial vendors and government entities.19 Finally, one of the primary duties of ' + 'the DAF AI T&E champion would be to formally adopt and promulgate DAF-wide guidance, such as the February 2022 DAF-MIT AI Accelerator Artificial Intelligence ' + 'Acquisition Guidebook.20 As noted previously, the committee expects DAF leaders to substantially underestimate the level of investments required to implement ' + 'digital modernization of 18 While there are many definitions, a digital twin is generally defined as a digital representation of a physical object ' + 'or system that can be used to simulate its real-world behavior and characteristics.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... <font class=\"btext\">Adversarial ' + 'attacks will occur at 21 One notable exception is the Air Force Ground-Based Strategic Deterrence \"Sentinel\" program, which has incorporated digital ' + 'modernization principles, to include the use of digital twins, since program inception.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... <font class=\"btext\">These ' + 'teams can help develop and update defenses against adversarial attacks while also supporting the development of offensive adversarial attack techniques ' + '-- much in the same way that \"red air\" has played an indispensable role in improving the effectiveness of DAF over the past 40 years, or in how cyber ' + '\"white hat\" and red teams have been honing the skills of the DAF cyber defenders and attackers over the past decade. Red teams represent a critical component ' + 'of AI test design and the overarching requirements process.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 92...</div><div class=foundblock>... <font class=\"btext\">The ' + 'DAF does not have the modeling and simulation architectures, synthetic environments, digital twins, or computational power needed to support developing, ' + 'testing and sustaining advanced AI-enabled systems. These deficiencies, 23 As noted by the 96 OG/CC, because Eglin AFB is a designated Major Range and ' + 'Test Facility Base (MRTFB)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=92\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 93...</div><div class=foundblock>... <font class=\"btext\">The ' + 'committee recommends storing and protecting data at all levels of classification within the purview of zero-trust network architecture and accounting for ' + 'data privacy when systems are trained on sensitive data. Recommendation 4-5: Building off the 2020 DoD Data Strategy, the Depart ment of the Air Force ' + 'should update and promulgate its data vision, strategy, and metrics-based implementation plan to explicitly recognize data as a \"first-class citizen.\" ' + 'These documents should include plans for the following: • Prioritizing investments in computation and storage resources and infrastructure to support ' + 'artificial intelligence (AI)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=93\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... <font class=\"btext\">Attempting ' + 'to drive systemic changes across the test community without fully understanding the nature and magnitude of the change required or failing to com municate ' + 'the rationale for change throughout the entire community can cause irreversible harm to the existing culture while simultaneously preventing leaders and ' + 'organizations from forging and sustaining a culture that can endure for the foreseeable future. For these reasons, it is critical to identify specific ' + 'aspects of the DAF test enterprise culture that need to be changed and why.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... <font class=\"btext\">The ' + 'committee recommends that DAF leaders communicate immediately to the Air and Space Forces both the importance of AI T&E and their commitment to establishing ' + 'a culture unique to AI T&E through the right combination of people, processes, and technology. At the same time, they should emphasize the value of preserving ' + 'the successful elements of the current DAF test community culture.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... <font class=\"btext\">, ' + 'AFRL, and AIA.25 Furthermore, the AFTC AI T&E curriculum should be developed by personnel with substantial AI and AI T&E experience, not only from within ' + 'the DAF but also, as appropriate, from industry and academia. The committee expects the test community will achieve better results this way rather than ' + 'relying primarily on retraining AFTC, AFOTEC, or USAFWC test personnel on AI principles and AI T&E processes, practices, and procedures.26 The committee ' + 'recommends that the DAF assess the utility of using the law school analogy for building a cadre of AI T&E personnel across the test enterprise.</font></div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... <font class=\"btext\">DAU is pivoting from a \"one-size-fits-all\" ' + 'certification framework to a component and workforce-centric, tailorable, continuous learning construct.31 The DoD 27 The committee suggests the AI ' + 'T&E champion, in coordination with the AFTC, AFOTEC, USAFWC, DAF CDAO, AFMC Digital Transformation Office (DTO) , and DAF Chief Experience Officer (CXO)</font></div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 98...</div><div class=foundblock>... <font class=\"btext\">In particular, the DAF and the DAF ' + 'AI champion should: • Provide AI education, training, and, where applicable, certifications to all personnel, from general officers and senior civilian ' + 'executives to entry-level personnel • Institute career-long tracking and management of personnel with specific AI and AI T&E skills • Place core ' + 'AI T&E training under the Air Force Test Center • Take advantage of existing AI-related education and training initiatives One of the most important ' + 'first steps is to survey the entire DAF workforce to determine as accurately as possible the current baseline of AI and AI T&E skills that exist in the ' + 'DAF today. The committee heard the resounding message from several speakers that such a baseline does not exist -- not for general AI skills or </font></div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=98\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... <font class=\"btext\">However, in the near term, the DAF ' + 'AI T&E champion, supported by DAF senior leaders, should use the survey results described above to coordinate across the entire DAF to help rebalance the ' + 'test force by shifting people with needed expertise into the test enterprise. At the same time, DAF test leaders should solicit volunteers from within ' + 'the test community to be trained specifically on AI T&E.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... <font ' + 'class=\"btext\">and AI testing and eval uation (T&E) skills across the DAF, develop and maintain a tiered approach 34 The \"National Security Commission ' + 'on AI Final Report\" includes several recommendations along these lines, to include the establish a new digital service academy and civilian national reserve ' + 'to grow tech talent.</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 101...</div><div class=foundblock>... <font ' + 'class=\"btext\">, and the AI education parts of Recommendation 4-6 would be implemented. Having established the current baseline of AI and AI T&E skills ' + 'across the DAF, this would also be the time frame where the DAF should develop a tiered approach to AI and AI T&E education and rebalance the test force ' + 'by shifting people with needed expertise into the test enterprise (Recommendation 4-7)</font></div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=101\');\">Read ' + 'the entire page → </a></div></td></tr>'; document.getElementById("texttitle").innerHTML = ''; document.getElementById("linkphrase").innerHTML = displayit; } var AllPhrases=new Array(); AllPhrases.push('daf_test'); AllPhrases.push('test_enterprise'); AllPhrases.push('test_community'); AllPhrases.push('daf_leaders'); AllPhrases.push('software_acquisition'); AllPhrases.push('red_teams'); AllPhrases.push('daf_cdao'); AllPhrases.push('security_commission'); AllPhrases.push('adversarial_attacks'); AllPhrases.push('air_university'); AllPhrases.push('digital_modernization'); AllPhrases.push('acquisition_pathway'); AllPhrases.push('culture_change'); AllPhrases.push('space_forces'); AllPhrases.push('failure_modes'); AllPhrases.push('osd_cdao'); AllPhrases.push('test_culture'); AllPhrases.push('test_organizations'); AllPhrases.push('space_force'); AllPhrases.push('intelligence_final'); AllPhrases.push('committee_expects'); AllPhrases.push('model_performance'); AllPhrases.push('workforce_development'); AllPhrases.push('education_strategy'); AllPhrases.push('authorization_act'); AllPhrases.push('traditional_daf'); AllPhrases.push('daf_aia'); AllPhrases.push('dod_data'); AllPhrases.push('operational_data'); AllPhrases.push('defense_acquisition'); // establish top phrases var TopPhrases=new Array(); for (x=0;x<30;x++){ TopPhrases.push(AllPhrases[x]); } function x_daf_test() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"daf test"</b> shown in context from 6 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_daf_test" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... , end-users, designated acquisition officials, and ' + 'the responsible <b>DAF test</b> organization. Such a dialogue will help developers and testers formulate T&E metrics and performance measures that best ' + 'match the end-users\' operational needs.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... When the ' + 'committee refers to requirements, it includes the need for DAF wide investments in enabling capabilities that support AI T&E at enterprise scale. Because ' + 'of AI\'s unique characteristics and the uncertainties associated with AI performance in operational environments, the committee recommends that the DAF ' + 'prioritize investments for digital modernization of the <b>DAF test</b> enterprise and for implementing an enterprise-level T&E architecture, enabled to ' + 'the maximum extent possible by the OSD CDAO,16 OSD DOT&E, OSD DASD(DT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... Those three ' + 'qualities describe the culture of today\'s <b>DAF test</b> enterprise. It is hard to argue with past successes.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... This sort ' + 'of culture change will be instrumental in accelerating the adoption and integration of AI across the DAF at speed and at scale. There are inherent dangers ' + 'in rushing to change the legacy <b>DAF test</b> culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 3 more on page 94</i></div></td></tr><tr bgcolor=white><td ' + 'valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The committee recommends that DAF leaders communicate immediately to the ' + 'Air and Space Forces both the importance of AI T&E and their commitment to establishing a culture unique to AI T&E through the right combination of people, ' + 'processes, and technology. At the same time, they should emphasize the value of preserving the successful elements of the current <b>DAF test</b> community ' + 'culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read the entire ' + 'page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... This may require deliberate ' + 'placement of AI T&E experts within operational and training units outside the traditional <b>DAF test</b> community. Some of these people may already be ' + 'test-certified (similar to how test pilots continually rotate through operational and training squadrons throughout their careers)</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... However, in the near term, the DAF AI T&E champion, supported ' + 'by DAF senior leaders, should use the survey results described above to coordinate across the entire DAF to help rebalance the test force by shifting people ' + 'with needed expertise into the test enterprise. At the same time, <b>DAF test</b> leaders should solicit volunteers from within the test community to be ' + 'trained specifically on AI T&E.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_test_enterprise() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"test enterprise"</b> shown in context from 6 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_test_enterprise" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... When the committee refers to requirements, it includes ' + 'the need for DAF wide investments in enabling capabilities that support AI T&E at enterprise scale. Because of AI\'s unique characteristics and the uncertainties ' + 'associated with AI performance in operational environments, the committee recommends that the DAF prioritize investments for digital modernization of the ' + 'DAF <b>test enterprise</b> and for implementing an enterprise-level T&E architecture, enabled to the maximum extent possible by the OSD CDAO,16 OSD DOT&E, ' + 'OSD DASD(DT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read the ' + 'entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... This departure ' + 'from established hardware test practices suggests the need for a DAF-wide <b>test enterprise</b> cultural shift, which in turn de pends on providing more ' + 'education and training on AI T&E and agile principles. The committee addresses this in more detail in the following Culture Change and Workforce Development ' + 'section.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read the entire ' + 'page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... Those three qualities ' + 'describe the culture of today\'s DAF <b>test enterprise</b>. It is hard to argue with past successes.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... Attempting ' + 'to drive systemic changes across the test community without fully understanding the nature and magnitude of the change required or failing to com municate ' + 'the rationale for change throughout the entire community can cause irreversible harm to the existing culture while simultaneously preventing leaders and ' + 'organizations from forging and sustaining a culture that can endure for the foreseeable future. For these reasons, it is critical to identify specific ' + 'aspects of the DAF <b>test enterprise</b> culture that need to be changed and why.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... , AFRL, ' + 'and AIA.25 Furthermore, the AFTC AI T&E curriculum should be developed by personnel with substantial AI and AI T&E experience, not only from within the ' + 'DAF but also, as appropriate, from industry and academia. The committee expects the test community will achieve better results this way rather than relying ' + 'primarily on retraining AFTC, AFOTEC, or USAFWC test personnel on AI principles and AI T&E processes, practices, and procedures.26 The committee recommends ' + 'that the DAF assess the utility of using the law school analogy for building a cadre of AI T&E personnel across the <b>test enterprise</b>.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... Once a baseline of personnel with AI and AI T&E experience ' + 'is established, the DAF AI T&E champion should coordinate with the applicable organizations and agencies to develop a tiered approach to AI implementations ' + 'and AI T&E-specific education, training, and training certification. This includes modifying existing programs to reflect the needs of the <b>test enterprise</b>.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... However, in the near term, the DAF AI T&E champion, supported ' + 'by DAF senior leaders, should use the survey results described above to coordinate across the entire DAF to help rebalance the test force by shifting people ' + 'with needed expertise into the <b>test enterprise</b>. At the same time, DAF test leaders should solicit volunteers from within the test community to be ' + 'trained specifically on AI T&E.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 101...</div><div class=foundblock>... , and ' + 'the AI education parts of Recommendation 4-6 would be implemented. Having established the current baseline of AI and AI T&E skills across the DAF, this ' + 'would also be the time frame where the DAF should develop a tiered approach to AI and AI T&E education and rebalance the test force by shifting people ' + 'with needed expertise into the <b>test enterprise</b> (Recommendation 4-7)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=101\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_test_community() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"test community"</b> shown in context from 7 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_test_community" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 79...</div><div class=foundblock>... will affect a significantly larger part of the DAF ' + 'than seems obvious at first glance, as the committee expects AI to be embedded throughout the entire DAF over the next decade. This chapter discusses the ' + 'actual scope of the impact of these advancements -- not only on the <b>test community</b> but also on the requirements processes and DAF culture.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=79\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... and established during system design and development. ' + 'One of the di lemmas the AI <b>test community</b> must grapple with is to understand when more traditional KPPs or NR-KPPs should apply, and when more ' + 'flexibility is required to avoid placing undue constraints on AI systems that are designed to meet end-user needs under operationally-relevant timelines.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... Attempting to drive systemic changes across the <b>test ' + 'community</b> without fully understanding the nature and magnitude of the change required or failing to com municate the rationale for change throughout ' + 'the entire community can cause irreversible harm to the existing culture while simultaneously preventing leaders and organizations from forging and sustaining ' + 'a culture that can endure for the foreseeable future. For these reasons, it is critical to identify specific aspects of the DAF test enterprise culture ' + 'that need to be changed and why.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The committee ' + 'recommends that DAF leaders communicate immediately to the Air and Space Forces both the importance of AI T&E and their commitment to establishing a culture ' + 'unique to AI T&E through the right combination of people, processes, and technology. At the same time, they should emphasize the value of preserving the ' + 'successful elements of the current DAF <b>test community</b> culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... , AFRL, ' + 'and AIA.25 Furthermore, the AFTC AI T&E curriculum should be developed by personnel with substantial AI and AI T&E experience, not only from within the ' + 'DAF but also, as appropriate, from industry and academia. The committee expects the <b>test community</b> will achieve better results this way rather than ' + 'relying primarily on retraining AFTC, AFOTEC, or USAFWC test personnel on AI principles and AI T&E processes, practices, and procedures.26 The committee ' + 'recommends that the DAF assess the utility of using the law school analogy for building a cadre of AI T&E personnel across the test enterprise.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... This may require deliberate placement of AI T&E experts ' + 'within operational and training units outside the traditional DAF <b>test community</b>. Some of these people may already be test-certified (similar to ' + 'how test pilots continually rotate through operational and training squadrons throughout their careers)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... The AIA ' + 'has compiled a list of AI educational resources for DoD personnel, which can be accessed with a common access card (CAC) .30 Similarly, the 96th Operations ' + 'Group Commander briefed the committee that the 96th is developing AI T&E educational programs for the <b>test community</b> to address the implications ' + 'of lethal autonomous weapon systems (LAWS)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... However, ' + 'in the near term, the DAF AI T&E champion, supported by DAF senior leaders, should use the survey results described above to coordinate across the entire ' + 'DAF to help rebalance the test force by shifting people with needed expertise into the test enterprise. At the same time, DAF test leaders should solicit ' + 'volunteers from within the <b>test community</b> to be trained specifically on AI T&E.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... , people ' + 'who have certain AI and AI T&E skills -- similar to how various career fields, including the air force <b>test community</b> -- identify special 32 ' + 'The CLSA is a self-paced, online program comprising a series of tests and exercises designed to evaluate an individual\'s knowledge of programming concepts ' + 'and techniques. Such a survey could also be used to gauge interest in entering the <b>test community</b> as an AI and AI T&E specialist.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_daf_leaders() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"daf leaders"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_daf_leaders" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... and PEOs, should provide standardized contract options ' + 'to address the need for TQD and machine-readable data, along with options for intellectual property (IP) protections and ownership of data rights and licenses ' + 'for both commercial vendors and government entities.19 Finally, one of the primary duties of the DAF AI T&E champion would be to formally adopt and promulgate ' + 'DAF-wide guidance, such as the February 2022 DAF-MIT AI Accelerator Artificial Intelligence Acquisition Guidebook.20 As noted previously, the committee ' + 'expects <b>DAF leaders</b> to substantially underestimate the level of investments required to implement digital modernization of 18 While there are ' + 'many definitions, a digital twin is generally defined as a digital representation of a physical object or system that can be used to simulate its real-world ' + 'behavior and characteristics.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 92...</div><div class=foundblock>... For cyber ' + 'sys tems, the AFCYBER incident response plan can trigger requirements for changes to fielded capabilities. <b>DAF leaders</b> should consider the advantages ' + 'and limitations of all these different processes -- as well as those in other DoD organizations and private sector companies -- when establishing new processes ' + 'and procedures that govern requests for urgent updates to fielded AI models.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=92\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 92...</div><div class=foundblock>... and is ' + 'funded through a \"pay to play\" model (as directed by the NDAA) , <b>DAF leaders</b> must address the disconnect between the timelines inherent in this ' + 'type of funding model, and the certainty of needing immediate funding for high-priority emerging AI T&E requirements.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=92\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The committee ' + 'recommends that <b>DAF leaders</b> communicate immediately to the Air and Space Forces both the importance of AI T&E and their commitment to establishing ' + 'a culture unique to AI T&E through the right combination of people, processes, and technology. At the same time, they should emphasize the value of preserving ' + 'the successful elements of the current DAF test community culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The designated ' + 'DAF AI T&E champion should be equally committed to long-term culture change and should be responsible for recommending changes to <b>DAF leaders</b> that ' + 'are designed to help forge a new AI T&E culture. The champion should also be accountable for following through on the decisions of <b>DAF leaders</b>.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 98...</div><div class=foundblock>... testing and evaluation (T&E) culture espoused by <b>DAF ' + 'leaders</b> and led by the AI T&E champion.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=98\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_software_acquisition() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"software acquisition"</b> shown in context from 2 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_software_acquisition" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... , major capability acquisition, <b>software acquisition</b>, ' + 'and defense business systems (DBS)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... This Instruction ' + 'addresses the use of the adaptive acquisition framework (AAF) in <b>software acquisition</b>.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 3 more on page 83</i></div></td></tr><tr bgcolor=white><td ' + 'valign=top><div class=fromblock>From page 85...</div><div class=foundblock>... 11 DoDI 5000.89 requires a test strategy when using the <b>software ' + 'acquisition</b> pathway, and notes that this pathway \"focuses on modern iterative software development techniques such as agile, lean, and development ' + 'security operations, which promise faster delivery of working code to the user. The goal of this <b>software acquisition</b> pathway is to achieve continuous ' + 'integration and continuous delivery to the maximum extent possible\" (p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=85\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_red_teams() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"red teams"</b> shown in context from 2 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_red_teams" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... These teams can help develop and update defenses against ' + 'adversarial attacks while also supporting the development of offensive adversarial attack techniques -- much in the same way that \"red air\" has played ' + 'an indispensable role in improving the effectiveness of DAF over the past 40 years, or in how cyber \"white hat\" and <b>red teams</b> have been honing ' + 'the skills of the DAF cyber defenders and attackers over the past decade. <b>Red teams</b> represent a critical component of AI test design and the overarching ' + 'requirements process.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... These teams ' + 'must be capable of emulating current and future peer competitor capabilities and performance and should be integrated into the entire AI life cycle. Furthermore, ' + 'the committee underscores the importance of not viewing AI <b>red teams</b> as entities that are completely separate from the AI T&E enterprise.22 Instead, ' + 'they should be integral to AI T&E, although independent, and focused on operational performance and mission resilience in the face of known and unknown ' + '-- but expected -- adversarial attacks, beginning with the presumption of attack at every stage of the AI life cycle, including cyberattacks, data manipulation, ' + 'and data corruption and poisoning (as discussed in the next chapter)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 2 more on page 91</i></div></td></tr><tr bgcolor=white><td ' + 'valign=top><div class=fromblock>From page 101...</div><div class=foundblock>... , and committing to establishing independent <b>red teams</b> (Recommendation ' + '4-4) can all be implemented quickly.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=101\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_daf_cdao() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"daf cdao"</b> shown in context from 5 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_daf_cdao" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... is a positive and important step, and the report committee ' + 'views the 96 OG/CC as one of the primary beneficiaries of this report. However, as currently constituted, the chief of AI test and operations for the <b>DAF ' + 'CDAO</b> does not have the authority to make the magnitude of changes across the DAF this committee believes necessary to enable AI T&E.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... Space Force Operational Test Agency (OTA) , the <b>DAF ' + 'CDAO</b>, and operational air, intelligence, C2, space, and cyber units.1 This process should in clude assessing what broader DAF-wide organizational ' + 'and governance changes are needed to reflect the differences between AI T&E and T&E for all other Air Force systems and capabilities.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 80</i></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page ' + '87...</div><div class=foundblock>... , and <b>DAF CDAO</b>, should be to establish \"a process for writing testable and verifiable AI requirement specifications ' + 'that characterize realistic operational performance,\" and to provide \"testing methodologies and metrics that enable evaluation of these requirements ' + '-- including principles of ethical and responsible AI, trustworthiness, robustness, and adversarial resilience.\"15 As noted above, the iterative and interactive ' + 'dialogue between end-users, testers, and the broader AI community will help operators and testers agree on request for proposal/request for information ' + '(RFP/RFI) and contract language, help endusers understand how AI performance will be assessed by testers, and help testers develop appropriate test metrics ' + 'and performance measures.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... , and <b>DAF ' + 'CDAO</b>. This should include major and near-term investments in modern AI stacks across AFTC, AFOTEC, and US AFWC (to include access to enterprise cloud-as-a-service ' + 'and platform-as-a-service [PaaS]</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... DAU is ' + 'pivoting from a \"one-size-fits-all\" certification framework to a component and workforce-centric, tailorable, continuous learning construct.31 The DoD ' + ' 27 The committee suggests the AI T&E champion, in coordination with the AFTC, AFOTEC, USAFWC, <b>DAF CDAO</b>, AFMC Digital Transformation Office (DTO) ' + ', and DAF Chief Experience Officer (CXO)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... and other ' + 'organizations, such as Air University and the DAF AIA, to develop and administer this DAF-wide survey. The DAF AI T&E champion, in coordination with the ' + '<b>DAF CDAO</b>, 96th Operations Group, AFPC, Air University, and the AIA, should consider taking the same approach used when developing the Air Force ' + 'Computer Language Self-Assessment (CLSA)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_security_commission() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"security commission"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_security_commission" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... See National <b>Security Commission</b> on Artificial ' + 'Intelligence (NSCAI) , 2021, National <b>Security Commission</b> on Artificial Intelligence Final Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/FullReport-Digital-1.pdf, ' + 'p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read the entire page ' + '→ </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... analysis, other studies ' + 'or analyses, or transition of rapidly fielded capability solutions. 8 NSCAI, 2021, National <b>Security Commission</b> on Artificial Intelligence Final ' + 'Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.pdf, p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... 15 ' + 'NSCAI, 2021, National <b>Security Commission</b> on Artificial Intelligence Final Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.pdf, ' + 'p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read the entire page ' + '→ </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... and AI testing and eval ' + ' uation (T&E) skills across the DAF, develop and maintain a tiered approach 34 The \"National <b>Security Commission</b> on AI Final Report\" includes ' + 'several recommendations along these lines, to include the establish a new digital service academy and civilian national reserve to grow tech talent.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... See NSCAI, 2021, National <b>Security Commission</b> ' + 'on Artificial Intelligence Final Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_adversarial_attacks() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"adversarial attacks"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_adversarial_attacks" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 82...</div><div class=foundblock>... At the same time, while overall model performance ' + 'is expected to improve over time as more operational data are ingested, absorption of more data could also lead to significant reductions in performance ' + 'if the new data are corrupted or poisoned or the AI model is subject to other forms of <b>adversarial attacks</b> (see Chapter 5)</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=82\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... Even with the rapid development of new AI capabilities ' + 'and the maturation of earlier AI-enabled systems that provide opportunities for rapid updates to fielded models, many AI systems today remain brittle. ' + 'Apart from the difficulties of field ing AI-enabled capabilities that perform as well in the operational environment as they do on the laboratory bench, ' + 'AI will be subject to corruption and <b>adversarial attacks</b> in the form of model or algorithm denial and deception, data poisoning, evasion attacks, ' + 'and cyberattacks among others.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... <b>Adversarial ' + 'attacks</b> will occur at 21 One notable exception is the Air Force Ground-Based Strategic Deterrence \"Sentinel\" program, which has incorporated ' + 'digital modernization principles, to include the use of digital twins, since program inception.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... These teams ' + 'can help develop and update defenses against <b>adversarial attacks</b> while also supporting the development of offensive adversarial attack techniques ' + '-- much in the same way that \"red air\" has played an indispensable role in improving the effectiveness of DAF over the past 40 years, or in how cyber ' + '\"white hat\" and red teams have been honing the skills of the DAF cyber defenders and attackers over the past decade. Red teams represent a critical component ' + 'of AI test design and the overarching requirements process.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... These teams ' + 'must be capable of emulating current and future peer competitor capabilities and performance and should be integrated into the entire AI life cycle. Furthermore, ' + 'the committee underscores the importance of not viewing AI red teams as entities that are completely separate from the AI T&E enterprise.22 Instead, they ' + 'should be integral to AI T&E, although independent, and focused on operational performance and mission resilience in the face of known and unknown -- but ' + 'expected -- <b>adversarial attacks</b>, beginning with the presumption of attack at every stage of the AI life cycle, including cyberattacks, data manipulation, ' + 'and data corruption and poisoning (as discussed in the next chapter)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 91</i></div></td></tr><tr bgcolor=white><td ' + 'valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... In particular, these include the lack of a clear delineation between DT ' + 'and OT for AI capabilities; the importance of and reliance on agile principles and adaptive T&E principles (AIOps, MLOps, or DevSecOps) instead of waterfall ' + 'development for AI systems; the centrality of data and high-end computing; the potential for a continuous data-based self-learning capability; the importance ' + 'and challenges of mission- and domain-specific adaptation for AI-enabled systems; probabilistic or statistically predictable behavior rather than deterministic ' + 'results; the effects and risks of dedicated <b>adversarial attacks</b> against AI models, at every phase from initial algorithm training through model ' + 'deployment and sustainment; the desire for AI explainability and auditability; and the need for continuous integration and continuous delivery (CI and ' + 'CD)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read the entire page ' + '→ </a></div></td></tr>' + '</tbody></table>' + ''; } function x_air_university() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"air university"</b> shown in context from 2 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_air_university" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... in coordination with <b>Air University</b> and OSD ' + 'CDAO -- should ensure that all levels of personnel have the appropriate training, from general officers and senior civilian executives to entry-level personnel.24 ' + 'CDAO now also has AI education initiatives with JHUAPL and Naval Postgraduate School (NPS) /Stanford.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... and other ' + 'organizations, such as <b>Air University</b> and the DAF AIA, to develop and administer this DAF-wide survey. The DAF AI T&E champion, in coordination ' + 'with the DAF CDAO, 96th Operations Group, AFPC, <b>Air University</b>, and the AIA, should consider taking the same approach used when developing the Air ' + 'Force Computer Language Self-Assessment (CLSA)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... program ' + 'in 2019. The CLSA, administered by <b>Air University</b>, allows DAF active duty, reserve, and civilian personnel to assess their knowledge and skills ' + 'in various computer programming languages.32 Modifying the CLSA to allow personnel to identify their AI and AI test-specific skills and any formal AI training ' + 'courses and certifications, while not perfect, is the best way to accelerate developing a DAF-wide baseline of personnel with AI and AI test-specific credentials.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 99</i></div></td></tr>' + '</tbody></table>' + ''; } function x_digital_modernization() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"digital modernization"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_digital_modernization" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... When the committee refers to requirements, it includes ' + 'the need for DAF wide investments in enabling capabilities that support AI T&E at enterprise scale. Because of AI\'s unique characteristics and the uncertainties ' + 'associated with AI performance in operational environments, the committee recommends that the DAF prioritize investments for <b>digital modernization</b> ' + 'of the DAF test enterprise and for implementing an enterprise-level T&E architecture, enabled to the maximum extent possible by the OSD CDAO,16 OSD DOT&E, ' + 'OSD DASD(DT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read the ' + 'entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... As more AI-enabled ' + 'weapon systems, especially AI-enabled autonomous weapon systems, are fielded across the Air Force and Space Force, there will be tremendous value in providing ' + 'dedicated T&E \"sandbox environments.\" Such environments will be vital in supporting T&E for systems in more operationallyrealistic settings and in providing ' + 'more insights into potential AI system limitations and failure modes while also allowing the appropriate assessment of individual system risks and the ' + 'risks associated with integration into a system-of-systems. <b>Digital modernization</b> includes building and sustaining data management pipelines (DMP)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... and PEOs, should provide standardized contract options ' + 'to address the need for TQD and machine-readable data, along with options for intellectual property (IP) protections and ownership of data rights and licenses ' + 'for both commercial vendors and government entities.19 Finally, one of the primary duties of the DAF AI T&E champion would be to formally adopt and promulgate ' + 'DAF-wide guidance, such as the February 2022 DAF-MIT AI Accelerator Artificial Intelligence Acquisition Guidebook.20 As noted previously, the committee ' + 'expects DAF leaders to substantially underestimate the level of investments required to implement <b>digital modernization</b> of 18 While there are ' + 'many definitions, a digital twin is generally defined as a digital representation of a physical object or system that can be used to simulate its real-world ' + 'behavior and characteristics.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... Adversarial ' + 'attacks will occur at 21 One notable exception is the Air Force Ground-Based Strategic Deterrence \"Sentinel\" program, which has incorporated <b>digital ' + 'modernization</b> principles, to include the use of digital twins, since program inception.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_acquisition_pathway() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"acquisition pathway"</b> shown in context from 2 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_acquisition_pathway" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... DoDI 5000.02 states explicitly that \"Programs executing ' + 'the software <b>acquisition pathway</b> are not subject to the Joint Capabilities Integration and Development System (JCIDS) , and will be handled as specifically ' + 'provided for by the Vice Chairman of the Joint Chiefs of Staff, in consultation with Under Secretary of Defense for Acquisition and Sustainment (USD(A&S)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... 3. It also notes that \"Programs executing the software ' + '<b>acquisition pathway</b> will not be treated as major defense acquisition programs,\" p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 83</i></div></td></tr><tr bgcolor=white><td ' + 'valign=top><div class=fromblock>From page 85...</div><div class=foundblock>... 11 DoDI 5000.89 requires a test strategy when using the software <b>acquisition ' + 'pathway</b>, and notes that this pathway \"focuses on modern iterative software development techniques such as agile, lean, and development security operations, ' + 'which promise faster delivery of working code to the user. The goal of this software <b>acquisition pathway</b> is to achieve continuous integration and ' + 'continuous delivery to the maximum extent possible\" (p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=85\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_culture_change() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"culture change"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_culture_change" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... This departure from established hardware test practices ' + 'suggests the need for a DAF-wide test enterprise cultural shift, which in turn de pends on providing more education and training on AI T&E and agile principles. ' + 'The committee addresses this in more detail in the following <b>Culture Change</b> and Workforce Development section.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 93...</div><div class=foundblock>... development ' + '• Widely expanding data collection and curation for the entire range of AI planning and scoping, designing, training, evaluation, and feed back activities ' + '• Investing in data simulators for AI training and testing • Adapting approaches and architectures developed in private industry for AI-based systems ' + '4.4 <b>CULTURE CHANGE</b> AND WORKFORCE DEVELOPMENT The concept of culture is much easier to experience viscerally than it is to define or even describe ' + 'adequately. Yet culture is very real.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=93\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... While the ' + 'committee cannot offer a \"recipe\" for <b>culture change</b>, it nonetheless believes that <b>culture change</b>s are neces sary to ensure AI T&E\'s ' + 'best practices, processes, and procedures are adopted as rapidly as possible across the DAF. As noted throughout this report, despite many commonalities ' + 'between traditional T&E and AI T&E, there are also notable differ ences.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... This sort ' + 'of <b>culture change</b> will be instrumental in accelerating the adoption and integration of AI across the DAF at speed and at scale. There are inherent ' + 'dangers in rushing to change the legacy DAF test culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... <b>Culture ' + 'change</b> begins at the top. Changing any culture depends on setting and adhering to a coherent vision that aligns strategies, actions, incentives, and ' + 'metrics.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read the entire ' + 'page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The designated DAF ' + 'AI T&E champion should be equally committed to long-term <b>culture change</b> and should be responsible for recommending changes to DAF leaders that are ' + 'designed to help forge a new AI T&E culture. The champion should also be accountable for following through on the decisions of DAF leaders.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_space_forces() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"space forces"</b> shown in context from 2 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_space_forces" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... official who reports to the Secretary of the Air Force, ' + 'is responsive to the Chiefs of the Air and <b>Space Forces</b>, and who has the necessary resources and authorities to implement DAF-wide changes. The ' + '2022 dual-hat designation of the 96th Operations Commander as the chief of AI test and operations for the DAF Chief Data and AI Office (CDAO)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... Finding 4-1: Currently, no single person below the level ' + 'of the Secretary or the Chiefs of the Air and <b>Space Forces</b> has the requisite authority to implement DAF-wide changes to successfully test and evaluate ' + 'AI-enabled systems. For this reason, the committee recommends that the Secretary of the Air Force formally designate an overall DAF AI T&E champion at ' + 'the general officer or senior executive service level in the DAF, and delegate to them the necessary authorities to make changes on behalf of the Secretary ' + 'and Service Chiefs.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The committee ' + 'recommends that DAF leaders communicate immediately to the Air and <b>Space Forces</b> both the importance of AI T&E and their commitment to establishing ' + 'a culture unique to AI T&E through the right combination of people, processes, and technology. At the same time, they should emphasize the value of preserving ' + 'the successful elements of the current DAF test community culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_failure_modes() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"failure modes"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_failure_modes" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 82...</div><div class=foundblock>... This \"probabilistic\" behavior is intrinsic to all ' + 'sensing systems. What is unique to many AI models is that their behavior under these data corruption and stochastic behavior scenarios are not well understood ' + 'at the theoretical level and often exhibits what is today seen as non-intuitive and brittle <b>failure modes</b>.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=82\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... standards ' + 'for testing intentional and unintentional <b>failure modes</b>; (2) exemplar datasets for benchmarking and evaluation, including robustness testing and ' + 'red teaming; and (3)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... It will ' + 'also depend on the level of risks that end-users are willing to accept based on their operational imperatives. Yet this requires the test-responsible organization ' + 'to communicate as transparently as possible to end-users measured and expected performance capabilities, system limitations, and possible <b>failure modes</b> ' + 'of AIenabled systems that users intend to accept for fielding.14 As the NSCAI recommended in its final report, one of the DAF\'s critical first steps, ' + 'led by the AI T&E champion in coordination with the OSD CDAO, OSD DOT&E, DASD(DT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... As more ' + 'AI-enabled weapon systems, especially AI-enabled autonomous weapon systems, are fielded across the Air Force and Space Force, there will be tremendous ' + 'value in providing dedicated T&E \"sandbox environments.\" Such environments will be vital in supporting T&E for systems in more operationallyrealistic ' + 'settings and in providing more insights into potential AI system limitations and <b>failure modes</b> while also allowing the appropriate assessment of ' + 'individual system risks and the risks associated with integration into a system-of-systems. Digital modernization includes building and sustaining data ' + 'management pipelines (DMP)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_osd_cdao() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"osd cdao"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_osd_cdao" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... The JAIC T&E division (now under the <b>OSD CDAO</b>) ' + 'refined Maven\'s processes, procedures, and practices and is publishing CDAO AI T&E playbooks and providing AI T&E frameworks to OSD DOT&E that the DAF ' + 'should consider adopting.12 This less constrained approach to AI requirements formulation introduces risks.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... \" question ' + 'should be one of the DAF AI T&E champion\'s initial top priorities, guided by discussions with the <b>OSD CDAO</b>, DOT&E, DASD(DT&E) , AF CDAO, AFMC DTO, ' + 'and other relevant DAF and joint AI test organizations and agencies.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... It will ' + 'also depend on the level of risks that end-users are willing to accept based on their operational imperatives. Yet this requires the test-responsible organization ' + 'to communicate as transparently as possible to end-users measured and expected performance capabilities, system limitations, and possible failure modes ' + 'of AIenabled systems that users intend to accept for fielding.14 As the NSCAI recommended in its final report, one of the DAF\'s critical first steps, ' + 'led by the AI T&E champion in coordination with the <b>OSD CDAO</b>, OSD DOT&E, DASD(DT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... When the ' + 'committee refers to requirements, it includes the need for DAF wide investments in enabling capabilities that support AI T&E at enterprise scale. Because ' + 'of AI\'s unique characteristics and the uncertainties associated with AI performance in operational environments, the committee recommends that the DAF ' + 'prioritize investments for digital modernization of the DAF test enterprise and for implementing an enterprise-level T&E architecture, enabled to the maximum ' + 'extent possible by the <b>OSD CDAO</b>,16 OSD DOT&E, OSD DASD(DT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... study, ' + '<b>OSD CDAO</b> is coordi nating with TRMC, DTE&A, and DOT&E to answer the following basic question: is DoD properly resourced to adequately test and ' + 'evaluate AI-enabled capabilities? This study is designed to systemati cally explore supply and demand for T&E of AI capabilities and identify gaps in ' + 'DoD infrastructure.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... in coordination ' + 'with Air University and <b>OSD CDAO</b> -- should ensure that all levels of personnel have the appropriate training, from general officers and senior civilian ' + 'executives to entry-level personnel.24 CDAO now also has AI education initiatives with JHUAPL and Naval Postgraduate School (NPS) /Stanford.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_test_culture() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"test culture"</b> shown in context from 1 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_test_culture" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... for fielded AI-enabled systems. The committee asserts ' + 'that the magnitude of these differences warrants devel oping a new culture, one that combines the best of the extant <b>test culture</b> with a new and ' + 'more risk-tolerant, agile, and adaptive mindset and approach to AI T&E.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... This sort ' + 'of culture change will be instrumental in accelerating the adoption and integration of AI across the DAF at speed and at scale. There are inherent dangers ' + 'in rushing to change the legacy DAF <b>test culture</b>.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 94</i></div></td></tr>' + '</tbody></table>' + ''; } function x_test_organizations() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"test organizations"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_test_organizations" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... Still, all short-term solutions will continue to evolve ' + 'over time through an itera tive, interactive process as air force end-users and personnel within responsible <b>test organizations</b> gain more experience ' + 'with writing AI-centric T&E require ments and with AI T&E processes and practices and as AI T&E becomes more automated and test results become more explainable. ' + 'The committee echoes the NSCAI\'s recommendation to the military services to \"establish a TEVV framework and culture that integrates testing as a continuous ' + 'part of requirements specifica tion, development, deployment, training, and maintenance and includes run-time monitoring of operational behavior.\"8 Section ' + '255 of the FY2020 National Defense Authorization Act (NDAA)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... \" question ' + 'should be one of the DAF AI T&E champion\'s initial top priorities, guided by discussions with the OSD CDAO, DOT&E, DASD(DT&E) , AF CDAO, AFMC DTO, and ' + 'other relevant DAF and joint AI <b>test organizations</b> and agencies.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... . However, ' + 'DAF responsible <b>test organizations</b> should be wary of allowing AI T&E to be \"held hostage\" when there are excessive delays in the parent weapon ' + 'system test sched ule during DT, OT, IOT&E, FOT&E, or live-fly test and evaluation (LFT&E)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_space_force() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"space force"</b> shown in context from 5 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_space_force" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... official who reports to the Secretary of the Air Force, ' + 'is responsive to the Chiefs of the Air and <b>Space Force</b>s, and who has the necessary resources and authorities to implement DAF-wide changes. The ' + '2022 dual-hat designation of the 96th Operations Commander as the chief of AI test and operations for the DAF Chief Data and AI Office (CDAO)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... Finding 4-1: Currently, no single person below the level ' + 'of the Secretary or the Chiefs of the Air and <b>Space Force</b>s has the requisite authority to implement DAF-wide changes to successfully test and evaluate ' + 'AI-enabled systems. For this reason, the committee recommends that the Secretary of the Air Force formally designate an overall DAF AI T&E champion at ' + 'the general officer or senior executive service level in the DAF, and delegate to them the necessary authorities to make changes on behalf of the Secretary ' + 'and Service Chiefs.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 80</i></div></td></tr><tr bgcolor=white><td ' + 'valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... As more AI-enabled weapon systems, especially AI-enabled autonomous weapon ' + 'systems, are fielded across the Air Force and <b>Space Force</b>, there will be tremendous value in providing dedicated T&E \"sandbox environments.\" Such ' + 'environments will be vital in supporting T&E for systems in more operationallyrealistic settings and in providing more insights into potential AI system ' + 'limitations and failure modes while also allowing the appropriate assessment of individual system risks and the risks associated with integration into ' + 'a system-of-systems. Digital modernization includes building and sustaining data management pipelines (DMP)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... The committee ' + 'recommends that DAF leaders communicate immediately to the Air and <b>Space Force</b>s both the importance of AI T&E and their commitment to establishing ' + 'a culture unique to AI T&E through the right combination of people, processes, and technology. At the same time, they should emphasize the value of preserving ' + 'the successful elements of the current DAF test community culture.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... 33 ' + 'Digital University is a joint venture started between the Air Force and <b>Space Force</b>, and is available to members of DoD. It provides access to Silicon ' + 'Valley-accredited technology training and fosters a community of learners.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... Given ' + 'the dearth of AI T&E expertise in the DAF today, the Air Force and <b>Space Force</b> can ill-afford to place personnel with these skills in positions ' + 'unrelated to AI and AI T&E (with normal exceptions granted for career development at more senior levels in the of ficer, enlisted, and civilian ranks)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_intelligence_final() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"intelligence final"</b> shown in context from 4 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_intelligence_final" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... See National Security Commission on Artificial Intelligence ' + '(NSCAI) , 2021, National Security Commission on Artificial <b>Intelligence Final</b> Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/FullReport-Digital-1.pdf, ' + 'p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read the entire page ' + '→ </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... analysis, other studies ' + 'or analyses, or transition of rapidly fielded capability solutions. 8 NSCAI, 2021, National Security Commission on Artificial <b>Intelligence Final</b> ' + 'Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.pdf, p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... 15 ' + 'NSCAI, 2021, National Security Commission on Artificial <b>Intelligence Final</b> Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.pdf, ' + 'p.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read the entire page ' + '→ </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... See NSCAI, 2021, National ' + 'Security Commission on Artificial <b>Intelligence Final</b> Report, Arlington, VA, https://www.nscai.gov/wp-content/uploads/2021/03/Full-Report-Digital-1.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_committee_expects() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"committee expects"</b> shown in context from 5 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_committee_expects" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 79...</div><div class=foundblock>... will affect a significantly larger part of the DAF ' + 'than seems obvious at first glance, as the <b>committee expects</b> AI to be embedded throughout the entire DAF over the next decade. This chapter discusses ' + 'the actual scope of the impact of these advancements -- not only on the test community but also on the requirements processes and DAF culture.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=79\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... and PEOs, should provide standardized contract options ' + 'to address the need for TQD and machine-readable data, along with options for intellectual property (IP) protections and ownership of data rights and licenses ' + 'for both commercial vendors and government entities.19 Finally, one of the primary duties of the DAF AI T&E champion would be to formally adopt and promulgate ' + 'DAF-wide guidance, such as the February 2022 DAF-MIT AI Accelerator Artificial Intelligence Acquisition Guidebook.20 As noted previously, the <b>committee ' + 'expects</b> DAF leaders to substantially underestimate the level of investments required to implement digital modernization of 18 While there are many ' + 'definitions, a digital twin is generally defined as a digital representation of a physical object or system that can be used to simulate its real-world ' + 'behavior and characteristics.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... , AFRL, ' + 'and AIA.25 Furthermore, the AFTC AI T&E curriculum should be developed by personnel with substantial AI and AI T&E experience, not only from within the ' + 'DAF but also, as appropriate, from industry and academia. The <b>committee expects</b> the test community will achieve better results this way rather than ' + 'relying primarily on retraining AFTC, AFOTEC, or USAFWC test personnel on AI principles and AI T&E processes, practices, and procedures.26 The committee ' + 'recommends that the DAF assess the utility of using the law school analogy for building a cadre of AI T&E personnel across the test enterprise.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 98...</div><div class=foundblock>... , which is viewed as a \"Weapons School for the acquisition ' + 'career field,\" includes an entire section on traditional T&E and another on emerging technologies. The <b>committee expects</b> that AFMC and AQIC will ' + 'be receptive to providing more advanced educa tion and training on AI T&E based on DAF, AFTC, AFOTEC, and USAFWC needs.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=98\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... As proposed, ' + 'the academy will provide scholarships for up to 5 years, in exchange for equivalent years of service in a civilian DoD position focused on digital technology ' + 'and cybersecurity. Over time, the <b>committee expects</b> that more computer science and AI degree-granting programs will increase the emphasis on AI ' + 'TEVV, perhaps even including TEVV as a specific subfield of study.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_model_performance() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"model performance"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_model_performance" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 82...</div><div class=foundblock>... At the same time, while overall <b>model performance</b> ' + 'is expected to improve over time as more operational data are ingested, absorption of more data could also lead to significant reductions in performance ' + 'if the new data are corrupted or poisoned or the AI model is subject to other forms of adversarial attacks (see Chapter 5)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=82\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... Members ' + 'of Project Maven \"translated\" esoteric T&E metrics into terms that were most relevant to operational end-users. Because formal requirements had not been ' + 'established for AI <b>model performance</b>, once the Maven team had completed data quality assurance, T&E on each model, integration testing in the Maven ' + 'Integration Lab, and live-fly testing, user acceptance of each trained model and follow-on updates to those fielded models, was based primarily on an agreement ' + 'between the Maven team and operational users that the models had demonstrated adequate performance under operational conditions (as compared to the baseline ' + 'performance achieved with existing, non-AI systems)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... model was ' + 'fielded, user feedback was instrumental in refining <b>model performance</b> through continuous integration and continuous delivery (CI/ CD)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... . This includes the importance of instrumenting fielded ' + 'systems to inform end-users of a potential adversarial attack or unexpected degradation in <b>model performance</b> (which may indicate an adversarial ' + 'attack)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91\');\">Read the entire ' + 'page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_workforce_development() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"workforce development"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_workforce_development" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... This departure from established hardware test practices ' + 'suggests the need for a DAF-wide test enterprise cultural shift, which in turn de pends on providing more education and training on AI T&E and agile principles. ' + 'The committee addresses this in more detail in the following Culture Change and <b>Workforce Development</b> section.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 93...</div><div class=foundblock>... development ' + '• Widely expanding data collection and curation for the entire range of AI planning and scoping, designing, training, evaluation, and feed back activities ' + '• Investing in data simulators for AI training and testing • Adapting approaches and architectures developed in private industry for AI-based systems ' + '4.4 CULTURE CHANGE AND <b>WORKFORCE DEVELOPMENT</b> The concept of culture is much easier to experience viscerally than it is to define or even describe ' + 'adequately. Yet culture is very real.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=93\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... <b>Workforce ' + 'development</b> is a critical component of the DAF-wide plan to introduce and sustain new AI T&E capabilities culture. In broad terms, <b>workforce development</b> ' + 'comprises training, education, certification, and talent management.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_education_strategy() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"education strategy"</b> shown in context from 1 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_education_strategy" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... (DoD Joint AI Center, 2020, Department of Defense ' + 'Artificial Intelligence <b>Education Strategy</b>, Washington, DC, https://www.ai.mil/docs/2020_DoD_AI_Training_and_Education_ Strategy_and_Infographic_10_27_20.pdf.) ' + 'See Chief Digital and Artificial Intelligence Office, 2023, \"Education & Training,\" https://www.ai.mil/education_training.html for descriptions of the ' + 'CDAO\'s AI training programs.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... 29 ' + 'As detailed on p. 7 in DoD AI <b>Education Strategy</b>, the six archetypes are Lead AI, Drive AI, Create AI, Employ AI, Facilitate AI, and Embed AI.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td><div class=foundblock>... <i>plus 1 more on page 97</i></div></td></tr>' + '</tbody></table>' + ''; } function x_authorization_act() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"authorization act"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_authorization_act" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... Still, all short-term solutions will continue to evolve ' + 'over time through an itera tive, interactive process as air force end-users and personnel within responsible test organizations gain more experience with ' + 'writing AI-centric T&E require ments and with AI T&E processes and practices and as AI T&E becomes more automated and test results become more explainable. ' + 'The committee echoes the NSCAI\'s recommendation to the military services to \"establish a TEVV framework and culture that integrates testing as a continuous ' + 'part of requirements specifica tion, development, deployment, training, and maintenance and includes run-time monitoring of operational behavior.\"8 Section ' + '255 of the FY2020 National Defense <b>Authorization Act</b> (NDAA)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... 28 ' + 'Section 256 of the National Defense <b>Authorization Act</b> (NDAA) for Fiscal Year 2020 directed the Secretary of Defense to \"develop a strategy for ' + 'educating service members in relevant occupational fields on matters related to artificial intelligence.\" It also directed the secretary to develop an ' + 'implementation plan.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... pdf. The ' + 'FY2023 National Defense <b>Authorization Act</b> contains provisions authorizing DoD to estab lish a cyber and digital service academy.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... McCain National Defense <b>Authorization Act</b> for ' + 'Fiscal Year 2019,\" H.R. 5515, 115th Congress (2017–2018)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_traditional_daf() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"traditional daf"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_traditional_daf" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... While end-user involvement and feedback are valu ' + 'able during the T&E of all systems, it is especially important during every stage of the AI life cycle due to general unfamiliarity with AI capabilities, ' + 'as well as AI\'s unique self-learning characteristics compared to all other <b>traditional DAF</b> hardware systems and software capabilities. The National ' + 'Institute of Standards and Technology\'s (NIST\'s)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... The level ' + 'and frequency of end-user or operator involvement in this process is another feature that distinguishes AI T&E from most <b>traditional DAF</b> hardware ' + 'testing practices. Addressing this foundational \"how much?</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... This may ' + 'require deliberate placement of AI T&E experts within operational and training units outside the <b>traditional DAF</b> test community. Some of these people ' + 'may already be test-certified (similar to how test pilots continually rotate through operational and training squadrons throughout their careers)</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_daf_aia() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"daf aia"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_daf_aia" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... provided either within the AFTC or at other designated ' + 'DAF or joint organizations, such as the <b>DAF AIA</b>, AFIT, AFRL, or the Defense Acquisition University (DAU)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... The JAIC ' + 'initiated an AI education pilot program in October 2020. The CDAO, in coordination with the <b>DAF AIA</b>, now offers a variety of AI training programs ' + 'and courses for personnel across DoD.</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... and other ' + 'organizations, such as Air University and the <b>DAF AIA</b>, to develop and administer this DAF-wide survey. The DAF AI T&E champion, in coordination ' + 'with the DAF CDAO, 96th Operations Group, AFPC, Air University, and the AIA, should consider taking the same approach used when developing the Air Force ' + 'Computer Language Self-Assessment (CLSA)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_dod_data() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"dod data"</b> shown in context from 2 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_dod_data" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... See also Department of Defense, 2020, <b>DoD Data</b> ' + 'Strategy, September 20, Washington, DC, https://media.defense.gov/2020/ Oct/08/2002514180/-1/-1/0/DoD-Data-Strategy.pdf. 20 Department of the Air Force, ' + '2022, Artificial Intelligence Acquisition Guidebook, Cambridge, MA: MIT, https://aia.mit.edu/wp-content/uploads/2022/02/AI-Acquisition-Guidebook_CAO14-Feb-2022.pdf.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 93...</div><div class=foundblock>... The committee recommends storing and protecting data at ' + 'all levels of classification within the purview of zero-trust network architecture and accounting for data privacy when systems are trained on sensitive ' + 'data. Recommendation 4-5: Building off the 2020 <b>DoD Data</b> Strategy, the Depart ment of the Air Force should update and promulgate its data vision, ' + 'strategy, and metrics-based implementation plan to explicitly recognize data as a \"first-class citizen.\" These documents should include plans for the ' + 'following: • Prioritizing investments in computation and storage resources and infrastructure to support artificial intelligence (AI)</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=93\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_operational_data() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"operational data"</b> shown in context from 1 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_operational_data" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 82...</div><div class=foundblock>... At the same time, while overall model performance ' + 'is expected to improve over time as more <b>operational data</b> are ingested, absorption of more data could also lead to significant reductions in performance ' + 'if the new data are corrupted or poisoned or the AI model is subject to other forms of adversarial attacks (see Chapter 5)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=82\');\">Read ' + 'the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } function x_defense_acquisition() { document.getElementById("texttitle").innerHTML = '<table width=100% vspace=0 hspace=0 cellpadding=1 cellspacing=0><tbody><tr><td align=left><div class="active-filter"><b>"defense acquisition"</b> shown in context from 3 pages. <a href="javascript:show_start();" class="clear-filter">x</a></div>' + '<img src="http://nap.nationalacademies.org/nap-cgi/webbug.cgi?referer=&dexterm_defense_acquisition" height=1 width=1></td>' + '<td align=right></td></tr></tbody></table>'; document.getElementById("text2").innerHTML = p_i_c_table + '<tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... <b>Defense acquisition</b> of services does not require ' + 'T&E policy and procedures. DoDI 5000.89 states that \"For non-major <b>defense acquisition</b> programs (MDAPs)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... 3. It also ' + 'notes that \"Programs executing the software acquisition pathway will not be treated as major <b>defense acquisition</b> programs,\" p.</div><div class=seepage><a ' + 'href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83\');\">Read the entire page → </a></div></td></tr><tr ' + 'bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... provided either within the AFTC or at other designated ' + 'DAF or joint organizations, such as the DAF AIA, AFIT, AFRL, or the <b>Defense Acquisition</b> University (DAU)</div><div class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96\');\">Read ' + 'the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... 31 ' + 'S Possehl, 2022, \"Test and Evaluation: The Change Is Here Today,\" <b>Defense Acquisition</b> Univer sity, February 1, https://www.dau.edu/library/defense-atl/blog/Test-and-Evaluation-change-today.</div><div ' + 'class=seepage><a href=\"javascript:poe(\'http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97\');\">Read the entire page → </a></div></td></tr>' + '</tbody></table>' + ''; } // --> </script> <script> // Back To Top control $(function() { var offset = 220; var duration = 500; $(window).scroll(function() { if ($(this).scrollTop() > offset) { $('.back-to-top').fadeIn(duration); } else { $('.back-to-top').fadeOut(duration); } }); $('.back-to-top').on('click', function(event) { event.preventDefault(); $('html, body').animate({scrollTop: 0}, duration); return false; }); }); </script> <a name="top" id="top"></a> <div class="container"><br> <div class="row"><a href="//www.nap.edu/catalog.php?record_id=27092">Test and Evaluation Challenges in Artificial Intelligence-Enabled Systems for the Department of the Air Force </a> (2023) / <span class="text-muted">Chapter Skim</span> </div> <div class="heading row"> <!-- left column --> <div class="col-xs-10 col-sm-3 col-md-2 col-lg-2 sibcol1"> <!-- cover image --> <div class="col-xs-4 col-sm-10 col-md-10 col-lg-10"> <a href="https://www.nap.edu/catalog.php?record_id=27092"><img src="//www.nap.edu/cover/27092/200" class="img-responsive img-thumbnail" border="0"></a> </div> <!-- /cover image --> <!-- buying_info --> <div class="col-xs-6 col-sm-10 col-md-10 col-lg-10"> <!-- buying_info --> <div id="leftcolumn"> <div class="print"> <a href="http://nap.nationalacademies.org/cart/cart.php?record_id=27092&isbn=0-309-70439-1&action=buy it&fs=1"class="buyit btn btn-default"> <span data-icon="" class="icon-menu"> </span><span class="webprice">$42.00</span> <br> Buy PAPERBACK</a> </div> <div class="results-actions"><a href="http://nap.nationalacademies.org/openbook.php?record_id=27092" class="readit">Read Online</a></div><div class="results-actions"> <a href="http://nap.nationalacademies.org/download.php?record_id=27092" class="dloadit"><span>Download Free PDF</span></a> </div> </div></div> </div> <!-- end buying_info --> <!-- /left column --> <!-- center column --> <div class="col-xs-10 col-sm-7 col-md-8 col-lg-8 sibcol2"> <!-- Book title and author info --> <div class="row chaphits"><span class="text-muted">Currently Skimming:</span> <h4>4 Evolution of Test and Evaluation in Future AI-Based DAF Systems <br><span class="pgcount">Pages 79-101</span></h4></div> <div> <div style="padding: 5px 5px 5px 5px;"> <form method="get" action="/skim.php"> <input type="hidden" name="record_id" value="27092"> <div> <select name='chap' onChange="this.form.submit()"> <option value='' selected>Select Another Chapter</option> <option value="1-16">Summary, pages 1-16</option> <option value="17-34">1 Introduction, pages 17-34</option> <option value="35-48">2 Definitions and Perspectives, pages 35-48</option> <option value="49-78">3 Test and Evaluation of DAF AI-Enabled Systems, pages 49-78</option> <option value="79-101">4 Evolution of Test and Evaluation in Future AI-Based..., pages 79-101</option> <option value="102-120">5 AI Technical Risks Under Operational Conditions, pages 102-120</option> <option value="121-136">6 Emerging AI Technologies and Future T&E Implications, pages 121-136</option> <option value="137-138">7 Concluding Thoughts, pages 137-138</option> <option value="139-140">Appendixes, pages 139-140</option> <option value="141-141">Appendix A: Statement of Task, pages 141-141</option> <option value="142-148">Appendix B: Public Meeting Agendas, pages 142-148</option> <option value="149-155">Appendix C: Committee Member Biographical Information, pages 149-155</option> <option value="156-162">Appendix D: Acronyms and Abbreviations, pages 156-162</option> <option value="163-180">Appendix E: Testing, Evaluating, and Assessing Artificial..., pages 163-180</option> </select> <!--<span class="input-group-btn"> <button type="submit" class="btn btn-primary btn-sm"> SKIM </button> </span>--> </div> <input name="act" value="nap" type="hidden"></form> </div> </div> <div class="row"> <div class="col-xs-10 chaphit_desc"> <p class="small">The Chapter Skim interface presents what we've algorithmically identified as the most significant single chunk of text within every page in the chapter. <br> Select <b>key terms</b> on the right to highlight them within pages of the chapter. </p></div> <div class="col-xs-10"> <br> <div id="texttitle" class="texttitle"> </div> <div class="col-xs-7 col-md-8 chapterlist"> <div id="text2" class="text2"> <table width="97%" bgcolor=darkgray cellpadding=5 cellspacing=1 id="skimresults"><tbody> <tr bgcolor=white><td valign=top><div class=fromblock>From page 79...</div><div class=foundblock>... <font class="btext">will affect a significantly larger part of the DAF than seems obvious at first glance, as the committee expects AI to be embedded throughout the entire DAF over the next decade. This chapter discusses the actual scope of the impact of these advancements -- not only on the test community but also on the requirements processes and DAF culture.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=79');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 80...</div><div class=foundblock>... <font class="btext">Space Force Operational Test Agency (OTA) , the DAF CDAO, and operational air, intelligence, C2, space, and cyber units.1 This process should in clude assessing what broader DAF-wide organizational and governance changes are needed to reflect the differences between AI T&E and T&E for all other Air Force systems and capabilities.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=80');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 81...</div><div class=foundblock>... <font class="btext">also recently established a digital transformation office (DTO) within AFMC; the AFMC commander used this similar approach to rectify the shortfall in implementing an effective digital strategy in the Air Force.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=81');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 82...</div><div class=foundblock>... <font class="btext">At the same time, while overall model performance is expected to improve over time as more operational data are ingested, absorption of more data could also lead to significant reductions in performance if the new data are corrupted or poisoned or the AI model is subject to other forms of adversarial attacks (see Chapter 5)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=82');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 83...</div><div class=foundblock>... <font class="btext">3. It also notes that "Programs executing the software acquisition pathway will not be treated as major defense acquisition programs," p.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=83');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 84...</div><div class=foundblock>... <font class="btext">Still, all short-term solutions will continue to evolve over time through an itera tive, interactive process as air force end-users and personnel within responsible test organizations gain more experience with writing AI-centric T&E require ments and with AI T&E processes and practices and as AI T&E becomes more automated and test results become more explainable. The committee echoes the NSCAI's recommendation to the military services to "establish a TEVV framework and culture that integrates testing as a continuous part of requirements specifica tion, development, deployment, training, and maintenance and includes run-time monitoring of operational behavior."8 Section 255 of the FY2020 National Defense Authorization Act (NDAA)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=84');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 85...</div><div class=foundblock>... <font class="btext"> 11 DoDI 5000.89 requires a test strategy when using the software acquisition pathway, and notes that this pathway "focuses on modern iterative software development techniques such as agile, lean, and development security operations, which promise faster delivery of working code to the user. The goal of this software acquisition pathway is to achieve continuous integration and continuous delivery to the maximum extent possible" (p.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=85');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 86...</div><div class=foundblock>... <font class="btext">Members of Project Maven "translated" esoteric T&E metrics into terms that were most relevant to operational end-users. Because formal requirements had not been established for AI model performance, once the Maven team had completed data quality assurance, T&E on each model, integration testing in the Maven Integration Lab, and live-fly testing, user acceptance of each trained model and follow-on updates to those fielded models, was based primarily on an agreement between the Maven team and operational users that the models had demonstrated adequate performance under operational conditions (as compared to the baseline performance achieved with existing, non-AI systems)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=86');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 87...</div><div class=foundblock>... <font class="btext">, and DAF CDAO, should be to establish "a process for writing testable and verifiable AI requirement specifications that characterize realistic operational performance," and to provide "testing methodologies and metrics that enable evaluation of these requirements -- including principles of ethical and responsible AI, trustworthiness, robustness, and adversarial resilience."15 As noted above, the iterative and interactive dialogue between end-users, testers, and the broader AI community will help operators and testers agree on request for proposal/request for information (RFP/RFI) and contract language, help endusers understand how AI performance will be assessed by testers, and help testers develop appropriate test metrics and performance measures.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=87');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 88...</div><div class=foundblock>... <font class="btext">When the committee refers to requirements, it includes the need for DAF wide investments in enabling capabilities that support AI T&E at enterprise scale. Because of AI's unique characteristics and the uncertainties associated with AI performance in operational environments, the committee recommends that the DAF prioritize investments for digital modernization of the DAF test enterprise and for implementing an enterprise-level T&E architecture, enabled to the maximum extent possible by the OSD CDAO,16 OSD DOT&E, OSD DASD(DT&E)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=88');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 89...</div><div class=foundblock>... <font class="btext">and PEOs, should provide standardized contract options to address the need for TQD and machine-readable data, along with options for intellectual property (IP) protections and ownership of data rights and licenses for both commercial vendors and government entities.19 Finally, one of the primary duties of the DAF AI T&E champion would be to formally adopt and promulgate DAF-wide guidance, such as the February 2022 DAF-MIT AI Accelerator Artificial Intelligence Acquisition Guidebook.20 As noted previously, the committee expects DAF leaders to substantially underestimate the level of investments required to implement digital modernization of 18 While there are many definitions, a digital twin is generally defined as a digital representation of a physical object or system that can be used to simulate its real-world behavior and characteristics.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=89');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 90...</div><div class=foundblock>... <font class="btext">Adversarial attacks will occur at 21 One notable exception is the Air Force Ground-Based Strategic Deterrence "Sentinel" program, which has incorporated digital modernization principles, to include the use of digital twins, since program inception.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=90');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 91...</div><div class=foundblock>... <font class="btext">These teams can help develop and update defenses against adversarial attacks while also supporting the development of offensive adversarial attack techniques -- much in the same way that "red air" has played an indispensable role in improving the effectiveness of DAF over the past 40 years, or in how cyber "white hat" and red teams have been honing the skills of the DAF cyber defenders and attackers over the past decade. Red teams represent a critical component of AI test design and the overarching requirements process.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=91');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 92...</div><div class=foundblock>... <font class="btext">The DAF does not have the modeling and simulation architectures, synthetic environments, digital twins, or computational power needed to support developing, testing and sustaining advanced AI-enabled systems. These deficiencies, 23 As noted by the 96 OG/CC, because Eglin AFB is a designated Major Range and Test Facility Base (MRTFB)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=92');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 93...</div><div class=foundblock>... <font class="btext">The committee recommends storing and protecting data at all levels of classification within the purview of zero-trust network architecture and accounting for data privacy when systems are trained on sensitive data. Recommendation 4-5: Building off the 2020 DoD Data Strategy, the Depart ment of the Air Force should update and promulgate its data vision, strategy, and metrics-based implementation plan to explicitly recognize data as a "first-class citizen." These documents should include plans for the following: • Prioritizing investments in computation and storage resources and infrastructure to support artificial intelligence (AI)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=93');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 94...</div><div class=foundblock>... <font class="btext">Attempting to drive systemic changes across the test community without fully understanding the nature and magnitude of the change required or failing to com municate the rationale for change throughout the entire community can cause irreversible harm to the existing culture while simultaneously preventing leaders and organizations from forging and sustaining a culture that can endure for the foreseeable future. For these reasons, it is critical to identify specific aspects of the DAF test enterprise culture that need to be changed and why.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=94');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 95...</div><div class=foundblock>... <font class="btext">The committee recommends that DAF leaders communicate immediately to the Air and Space Forces both the importance of AI T&E and their commitment to establishing a culture unique to AI T&E through the right combination of people, processes, and technology. At the same time, they should emphasize the value of preserving the successful elements of the current DAF test community culture.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=95');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 96...</div><div class=foundblock>... <font class="btext">, AFRL, and AIA.25 Furthermore, the AFTC AI T&E curriculum should be developed by personnel with substantial AI and AI T&E experience, not only from within the DAF but also, as appropriate, from industry and academia. The committee expects the test community will achieve better results this way rather than relying primarily on retraining AFTC, AFOTEC, or USAFWC test personnel on AI principles and AI T&E processes, practices, and procedures.26 The committee recommends that the DAF assess the utility of using the law school analogy for building a cadre of AI T&E personnel across the test enterprise.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=96');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 97...</div><div class=foundblock>... <font class="btext">DAU is pivoting from a "one-size-fits-all" certification framework to a component and workforce-centric, tailorable, continuous learning construct.31 The DoD 27 The committee suggests the AI T&E champion, in coordination with the AFTC, AFOTEC, USAFWC, DAF CDAO, AFMC Digital Transformation Office (DTO) , and DAF Chief Experience Officer (CXO)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=97');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 98...</div><div class=foundblock>... <font class="btext">In particular, the DAF and the DAF AI champion should: • Provide AI education, training, and, where applicable, certifications to all personnel, from general officers and senior civilian executives to entry-level personnel • Institute career-long tracking and management of personnel with specific AI and AI T&E skills • Place core AI T&E training under the Air Force Test Center • Take advantage of existing AI-related education and training initiatives One of the most important first steps is to survey the entire DAF workforce to determine as accurately as possible the current baseline of AI and AI T&E skills that exist in the DAF today. The committee heard the resounding message from several speakers that such a baseline does not exist -- not for general AI skills or </font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=98');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 99...</div><div class=foundblock>... <font class="btext">However, in the near term, the DAF AI T&E champion, supported by DAF senior leaders, should use the survey results described above to coordinate across the entire DAF to help rebalance the test force by shifting people with needed expertise into the test enterprise. At the same time, DAF test leaders should solicit volunteers from within the test community to be trained specifically on AI T&E.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=99');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 100...</div><div class=foundblock>... <font class="btext">and AI testing and eval uation (T&E) skills across the DAF, develop and maintain a tiered approach 34 The "National Security Commission on AI Final Report" includes several recommendations along these lines, to include the establish a new digital service academy and civilian national reserve to grow tech talent.</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=100');">Read the entire page → </a></div></td></tr><tr bgcolor=white><td valign=top><div class=fromblock>From page 101...</div><div class=foundblock>... <font class="btext">, and the AI education parts of Recommendation 4-6 would be implemented. Having established the current baseline of AI and AI T&E skills across the DAF, this would also be the time frame where the DAF should develop a tiered approach to AI and AI T&E education and rebalance the test force by shifting people with needed expertise into the test enterprise (Recommendation 4-7)</font> </div><div class=seepage><a href="javascript:poe('http://nap.nationalacademies.org/openbook.php?record_id=27092&page=101');">Read the entire page → </a></div></td></tr> </table> </div> </div> <div class="col-xs-3 col-md-2 pull-right termlist"> <p class="small">Key Terms</p> <li class="xy"><a href="javascript:x_daf_test();">daf test</a></li><li class="xy"><a href="javascript:x_test_enterprise();">test enterprise</a></li><li class="xy"><a href="javascript:x_test_community();">test community</a></li><li class="xy"><a href="javascript:x_daf_leaders();">daf leaders</a></li><li class="xy"><a href="javascript:x_software_acquisition();">software acquisition</a></li><div class=ider><br></div> <li class="xy"><a href="javascript:x_red_teams();">red teams</a></li><li class="xy"><a href="javascript:x_daf_cdao();">daf cdao</a></li><li class="xy"><a href="javascript:x_security_commission();">security commission</a></li><li class="xy"><a href="javascript:x_adversarial_attacks();">adversarial attacks</a></li><li class="xy"><a href="javascript:x_air_university();">air university</a></li><div class=ider><br></div> <li class="xy"><a href="javascript:x_digital_modernization();">digital modernization</a></li><li class="xy"><a href="javascript:x_acquisition_pathway();">acquisition pathway</a></li><li class="xy"><a href="javascript:x_culture_change();">culture change</a></li><li class="xy"><a href="javascript:x_space_forces();">space forces</a></li><li class="xy"><a href="javascript:x_failure_modes();">failure modes</a></li><div class=ider><br></div> <li class="xy"><a href="javascript:x_osd_cdao();">osd cdao</a></li><li class="xy"><a href="javascript:x_test_culture();">test culture</a></li><li class="xy"><a href="javascript:x_test_organizations();">test organizations</a></li><li class="xy"><a href="javascript:x_space_force();">space force</a></li><li class="xy"><a href="javascript:x_intelligence_final();">intelligence final</a></li><div class=ider><br></div> <li class="xy"><a href="javascript:x_committee_expects();">committee expects</a></li><li class="xy"><a href="javascript:x_model_performance();">model performance</a></li><li class="xy"><a href="javascript:x_workforce_development();">workforce development</a></li><li class="xy"><a href="javascript:x_education_strategy();">education strategy</a></li><li class="xy"><a href="javascript:x_authorization_act();">authorization act</a></li><div class=ider><br></div> <li class="xy"><a href="javascript:x_traditional_daf();">traditional daf</a></li><li class="xy"><a href="javascript:x_daf_aia();">daf aia</a></li><li class="xy"><a href="javascript:x_dod_data();">dod data</a></li><li class="xy"><a href="javascript:x_operational_data();">operational data</a></li><li class="xy"><a href="javascript:x_defense_acquisition();">defense acquisition</a></li> </div> <div class="col-xs-8 chapskim-nav"> <!-- use previous chapter phrase --> <div class="pull-left prev "><a href="/skim.php?record_id=27092&chap=49-78&act=nap" class="btn btn-default">← Previous Chapter Skim</a></div> <!-- use next chapter phrase --> <div class="pull-right next"><a href="/skim.php?record_id=27092&chap=102-120&act=nap" class="btn btn-default">Next Chapter Skim →</a></div> </div> <div class="clearfix"><br clear="all"></div><div class="clearfix"><br clear="all"></div> </div> <!-- / chapterlist and termlist--> <div class="col-xs-10" > <p class="small">This material may be derived from roughly machine-read images, and so is provided only to facilitate research.<br> <a href="//www.nap.edu/about/skimit.html">More information on Chapter Skim</a> is available.</p> <br><br> </div> </div> </div> </div> </div> <br><br> <!--Footer --> </body></html> <br> </div> <!--start NAP footer --> <footer class="global-footer" style="background: #211747; border-top: none" id="global-footer-nap"> <div class="container"> <div class="global-footer__orgs"> <div class="global-footer__logo"> <a href="https://nap.nationalacademies.org"><img src="/images/footer-nap-text.png" alt="National Academies Press: Washington, DC | Home" class="img-responsive"></a> </div> <div> <form id="subscribe-form" action="/updates/" method="POST"> <input type="hidden" name="action" value="subscribe"> <div class="input-group"> <label for="email" class="sr-only">Email Address</label> <input type="hidden" name="SOURCE" value="homepage"> <input class="subscribe-input form-control" type="text" name="EMAIL" id="email" placeholder="Sign up for e-mail updates" title="Enter your e-mail address to subscribe to updates from NAP."> <span class="subscribe-button"> <button type="submit" class="btn btn-subscribe">Subscribe</button> </span> </div> </form></div></div> <div class="global-footer__links"> <ul class="global-footer__nav"> <li> <h6>NAP Quick Links</h6> <ul> <li><a href="/">Home</a></li> <li><a href="/content/about-the-national-academies-press">About</a></li> <li><a href="https://notes.nap.edu/">Notes from NAP</a></li> </ul> <h6>Request A</h6> <ul> <li><a href="/content/request-desk-copies">Desk Copy</a> </li> <li><a href="/content/request/exam_copy.html">Exam Copy </a></li> <li><a href="/content/request-review-copies">Review Copy</a></li> <li><a href="/content/request-rights-and-permissions">Reprint Permission</a></li> </ul> </li> <li> <h6>Browse Publications</h6> <ul> <li><a href="/author">By Division</a></li> <li><a href="/new/">By New Releases</a></li> <li><a href="/collection">By Searchable Collections</a></li> <li><a href="/topic">By Topic</a></li> </ul> </li> <li> <h6>NAP Ordering</h6> <ul> <li><a href="/content/contact-customer-service">Contact Customer Service</a></li> </ul> <h6>Using NAP.edu</h6> <ul> <li><a href="/content/searching-nap-publications">Tips for Searching</a> </li> <li><a href="/content/using-mynap">Help With MyNAP</a></li> <li><a href="/content/about-pdfs">Downloading and Reading PDFs</a> </li> </ul> </li> <li> <h6>NAP Information</h6> <ul> <li><a href="/content/help-with-discounts">Discounts</a></li> <li><a href="/content/information-for-book-trade-customers">Book Trade Customers</a></li> <li><a href="/content/request-rights-and-permissions">Rights and Permissions</a></li> <li><a href="/content/international-ordering-and-distribution">International Ordering and Distribution</a></li> <li><a href="/content/translation-rights">Translation Rights</a></li> </ul></li> </ul> </div></div></footer> <!--start NASEM footer --> <footer class="global-footer" style="background: #666; border-top: none"><div class="container"><div class="global-footer__orgs"><div class="global-footer__logo"> <a aria-label="Logo of The National Academies" href="https://www.nationalacademies.org/home"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 315.11 58.33"><defs><style>.cls-1{fill:#fff;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><polygon class="cls-1" points="3.73 7.55 20.84 25.49 20.84 1.5 17.81 1.5 17.81 18.08 0.7 0.14 0.7 24.06 3.73 24.06 3.73 7.55"/><polygon class="cls-1" points="28.49 24.06 31.84 24.06 38.98 7.25 45.92 23.87 45.99 24.06 49.34 24.06 39.05 0 28.49 24.06"/><polygon class="cls-1" points="153.23 24.06 156.58 24.06 163.72 7.25 170.66 23.87 170.73 24.06 174.08 24.06 163.79 0 153.23 24.06"/><polygon class="cls-1" points="59.72 24.06 62.76 24.06 62.76 4.41 69.52 4.41 69.52 1.5 52.96 1.5 52.96 4.41 59.72 4.41 59.72 24.06"/><rect class="cls-1" x="78.85" y="1.5" width="3.03" height="22.55"/><path class="cls-1" d="M115.69,12.8a11.72,11.72,0,0,0-23.44,0,11.72,11.72,0,0,0,23.44,0m-3.09,0A8.63,8.63,0,1,1,104,4a8.7,8.7,0,0,1,8.63,8.76"/><polygon class="cls-1" points="128.29 7.55 145.4 25.49 145.4 1.5 142.37 1.5 142.37 18.08 125.25 0.14 125.25 24.06 128.29 24.06 128.29 7.55"/><polygon class="cls-1" points="184.65 1.5 181.62 1.5 181.62 24.06 193.7 24.06 193.7 21.16 184.65 21.16 184.65 1.5"/><path class="cls-1" d="M37.15,34.08A11.48,11.48,0,0,0,25.37,45.8,11.36,11.36,0,0,0,37,57.38a11.77,11.77,0,0,0,6.79-2.19l.13-.09V51.2l-.51.51A9.77,9.77,0,0,1,41,53.5a8.9,8.9,0,0,1-3.95,1A8.75,8.75,0,0,1,37,37a9.09,9.09,0,0,1,3.66.82,10.71,10.71,0,0,1,2.78,2l.51.5V36.4l-.12-.09a10.62,10.62,0,0,0-6.67-2.23"/><path class="cls-1" d="M91.91,37.08a12.19,12.19,0,0,0-8.29-2.64H78.85V57h4.74c3.72,0,6-.73,8.22-2.61a10.75,10.75,0,0,0,4-8.65,11,11,0,0,0-3.86-8.65m-2.09,15.1a9.11,9.11,0,0,1-6.1,1.91H81.88V37.34h1.84c5.44,0,9,3.29,9,8.39a8.09,8.09,0,0,1-2.85,6.45"/><polygon class="cls-1" points="104.99 56.99 118.4 56.99 118.4 54.09 108.02 54.09 108.02 46.16 117.11 46.16 117.11 43.26 108.02 43.26 108.02 37.34 118.4 37.34 118.4 34.44 104.99 34.44 104.99 56.99"/><polygon class="cls-1" points="176.17 56.99 189.58 56.99 189.58 54.09 179.2 54.09 179.2 46.16 188.28 46.16 188.28 43.26 179.2 43.26 179.2 37.34 189.58 37.34 189.58 34.44 176.17 34.44 176.17 56.99"/><polygon class="cls-1" points="138.88 51.34 130.31 32.49 126.44 56.99 129.55 56.99 131.62 42.47 138.84 58.33 146.15 42.47 148.13 56.99 151.23 56.99 147.61 32.45 138.88 51.34"/><rect class="cls-1" x="161.31" y="34.44" width="3.03" height="22.55"/><path class="cls-1" d="M209,46.15A10.78,10.78,0,0,0,205.49,44l-2.12-.92a6.8,6.8,0,0,1-2.21-1.33,2,2,0,0,1-.73-1.69A3.07,3.07,0,0,1,203.82,37a3.62,3.62,0,0,1,2.28.69,4.87,4.87,0,0,1,1.2,1.55l.15.3,2.45-1.62-.12-.24a7,7,0,0,0-2.24-2.55,6.34,6.34,0,0,0-3.69-1c-3.8,0-6.45,2.45-6.45,6a4.86,4.86,0,0,0,1.6,3.8,9.51,9.51,0,0,0,2.84,1.78l2.15,1a9.5,9.5,0,0,1,2.56,1.46,3,3,0,0,1,1,2.35,4.2,4.2,0,0,1-7.06,2.92,4.93,4.93,0,0,1-1.24-2.64l-.05-.33-3,.82.06.28a8.32,8.32,0,0,0,2.13,4,6.74,6.74,0,0,0,5,1.87,6.91,6.91,0,0,0,7.24-7A5.58,5.58,0,0,0,209,46.15"/><polygon class="cls-1" points="0 56.99 3.35 56.99 10.49 40.18 17.43 56.81 17.5 56.99 20.86 56.99 10.56 32.93 0 56.99"/><polygon class="cls-1" points="50.21 56.99 53.57 56.99 60.71 40.18 67.64 56.81 67.72 56.99 71.07 56.99 60.78 32.93 50.21 56.99"/><path class="cls-1" d="M233.74,2.1a3.34,3.34,0,0,0-.93-.11,2.53,2.53,0,0,0-1.52.45,1.48,1.48,0,0,0-.63,1.28,2,2,0,0,0,.54,1.44,7.94,7.94,0,0,0,1.64,1.26,13.5,13.5,0,0,1,1.26.89,3.76,3.76,0,0,1,.83,1,2.62,2.62,0,0,1,.33,1.33,3.06,3.06,0,0,1-.5,1.74,3.26,3.26,0,0,1-1.4,1.18,5.06,5.06,0,0,1-2.09.41,8.3,8.3,0,0,1-1.94-.22,3.51,3.51,0,0,1-1.24-.46l.34-2.18h.92l.09,1.62a2.53,2.53,0,0,0,.81.33,3.86,3.86,0,0,0,1,.13,2.93,2.93,0,0,0,1.77-.53,1.68,1.68,0,0,0,.73-1.47,1.91,1.91,0,0,0-.49-1.34,6.42,6.42,0,0,0-1.49-1.16c-.52-.34-1-.65-1.32-.94a3.76,3.76,0,0,1-.9-1,2.64,2.64,0,0,1-.37-1.4,2.64,2.64,0,0,1,.5-1.62,3.23,3.23,0,0,1,1.35-1.08,4.58,4.58,0,0,1,1.86-.37,7,7,0,0,1,1.72.18,3.31,3.31,0,0,1,1.09.45l-.35,2h-.83l-.12-1.47a2.39,2.39,0,0,0-.68-.3"/><path class="cls-1" d="M241,5a2,2,0,0,0-1.35.54,3.66,3.66,0,0,0-.92,1.52,7,7,0,0,0-.33,2.24,3.52,3.52,0,0,0,.43,1.95,1.39,1.39,0,0,0,1.24.66,2.38,2.38,0,0,0,1.1-.24,5.05,5.05,0,0,0,1-.72l.3.57a3.84,3.84,0,0,1-1.14.95,3.43,3.43,0,0,1-1.76.43,2.44,2.44,0,0,1-2-.91,4,4,0,0,1-.73-2.54A6.43,6.43,0,0,1,237.34,7a5.3,5.3,0,0,1,1.55-2.08,3.74,3.74,0,0,1,2.42-.83,5,5,0,0,1,.93.09,3.09,3.09,0,0,1,.81.26l-.42,1.49-.26-.09A2.53,2.53,0,0,0,241,5"/><path class="cls-1" d="M246.1,11.61a.56.56,0,0,0,.07.29.21.21,0,0,0,.18.1.65.65,0,0,0,.34-.11,2.64,2.64,0,0,0,.44-.35l.2.37a2.88,2.88,0,0,1-.68.67,1.71,1.71,0,0,1-1.08.35,1.14,1.14,0,0,1-.78-.23.8.8,0,0,1-.22-.72c0-.25.15-1.25.4-3,.14-.93.31-2.16.52-3.7l-1-.35.09-.52,2.18-.36.34.17-1,7.22ZM246,2.15a.84.84,0,0,1-.23-.59,1.13,1.13,0,0,1,.3-.79.94.94,0,0,1,.71-.32.79.79,0,0,1,.61.24.77.77,0,0,1,.23.57,1.15,1.15,0,0,1-.29.81,1,1,0,0,1-.77.33.75.75,0,0,1-.56-.25"/><path class="cls-1" d="M254,12.43a3.75,3.75,0,0,1-1.95.52,3,3,0,0,1-1.72-.47,2.74,2.74,0,0,1-1-1.23A4,4,0,0,1,249,9.62a6.35,6.35,0,0,1,.59-2.75,5.08,5.08,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.27,2.27,0,0,1,1.64.52A1.79,1.79,0,0,1,255.72,6a2.34,2.34,0,0,1-.89,1.84,5.79,5.79,0,0,1-2.08,1.12,8.1,8.1,0,0,1-2.15.4,3.42,3.42,0,0,0,.37,1.87,1.48,1.48,0,0,0,1.4.79,3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57A4,4,0,0,1,254,12.43m-2.11-7A4.1,4.1,0,0,0,251,6.88a5.9,5.9,0,0,0-.39,1.69,5.31,5.31,0,0,0,1.62-.33,4,4,0,0,0,1.41-.87,1.76,1.76,0,0,0,.57-1.28,1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/><path class="cls-1" d="M261.24,4.62A3.14,3.14,0,0,1,263,4.06a1.67,1.67,0,0,1,1.32.52,2.61,2.61,0,0,1,.46,1.71,15.44,15.44,0,0,1-.34,2.34l-.06.36-.13.74a11.51,11.51,0,0,0-.24,1.85c0,.28.06.42.2.42a1.22,1.22,0,0,0,.72-.39l.18.4a1.54,1.54,0,0,1-.32.36,2.92,2.92,0,0,1-.61.39,1.73,1.73,0,0,1-.79.17.84.84,0,0,1-1-.93,12.88,12.88,0,0,1,.27-1.88l.14-.73.06-.37c.1-.55.19-1,.25-1.45a8.17,8.17,0,0,0,.09-1.16,1.67,1.67,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.33,2.33,0,0,0-1.37.61,5.75,5.75,0,0,0-1.3,1.47l-.75,5.54h-1.46l1-7.43L257.62,5l.12-.58,1.88-.34.32.17-.2,1.84a5.72,5.72,0,0,1,1.5-1.45"/><path class="cls-1" d="M270.77,5a2,2,0,0,0-1.35.54,3.66,3.66,0,0,0-.92,1.52,7,7,0,0,0-.33,2.24,3.52,3.52,0,0,0,.43,1.95,1.39,1.39,0,0,0,1.24.66,2.38,2.38,0,0,0,1.1-.24,5.05,5.05,0,0,0,1-.72l.3.57a3.84,3.84,0,0,1-1.14.95,3.43,3.43,0,0,1-1.76.43,2.44,2.44,0,0,1-2-.91,4,4,0,0,1-.73-2.54A6.43,6.43,0,0,1,267.13,7a5.3,5.3,0,0,1,1.55-2.08,3.76,3.76,0,0,1,2.42-.83,5,5,0,0,1,.93.09,3,3,0,0,1,.81.26l-.42,1.49-.26-.09A2.53,2.53,0,0,0,270.77,5"/><path class="cls-1" d="M278.66,12.43a3.75,3.75,0,0,1-2,.52,3,3,0,0,1-1.71-.47,2.7,2.7,0,0,1-1-1.23,4.19,4.19,0,0,1-.32-1.63,6.35,6.35,0,0,1,.59-2.75,5.17,5.17,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.52A1.82,1.82,0,0,1,280.38,6a2.34,2.34,0,0,1-.89,1.84,5.79,5.79,0,0,1-2.08,1.12,8,8,0,0,1-2.15.4,3.51,3.51,0,0,0,.37,1.87A1.48,1.48,0,0,0,277,12a3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.27,1.11m-2.11-7a4.25,4.25,0,0,0-.87,1.45,6.25,6.25,0,0,0-.39,1.69,5.44,5.44,0,0,0,1.63-.33,4.05,4.05,0,0,0,1.4-.87,1.76,1.76,0,0,0,.57-1.28,1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/><path class="cls-1" d="M286.18,5.13a2.75,2.75,0,0,0-1.09-.21,1.85,1.85,0,0,0-1.13.31,1,1,0,0,0-.43.89,1.43,1.43,0,0,0,.34.93,6.08,6.08,0,0,0,1,.88,10.43,10.43,0,0,1,.88.74,3.1,3.1,0,0,1,.6.76,2.15,2.15,0,0,1,.26,1.05,2.09,2.09,0,0,1-.9,1.81,3.71,3.71,0,0,1-2.22.64,6.32,6.32,0,0,1-1.37-.15,2.1,2.1,0,0,1-.9-.34l.24-1.06a3.41,3.41,0,0,0,.88.44,3.37,3.37,0,0,0,1.18.21,2.21,2.21,0,0,0,1.15-.29,1,1,0,0,0,.48-.92,1.37,1.37,0,0,0-.38-1,6.83,6.83,0,0,0-1.1-.89,3.07,3.07,0,0,1-1.58-2.5,2.16,2.16,0,0,1,.4-1.27,2.64,2.64,0,0,1,1.09-.84,3.81,3.81,0,0,1,1.54-.3,6.32,6.32,0,0,1,1.13.1,4.07,4.07,0,0,1,.8.19l-.26,1.17a1.55,1.55,0,0,0-.6-.39"/><polygon class="cls-1" points="229.34 23.43 236.61 23.43 236.51 25.87 235.6 25.87 235.35 24.25 231.87 24.13 231.29 28.48 234.85 28.36 234.74 29.51 231.21 29.38 230.58 34.05 234.91 33.87 235.77 32.01 236.63 32.18 236.01 34.84 227.85 34.84 227.91 34.24 229.06 34.05 230.29 24.25 229.24 24.09 229.34 23.43"/><path class="cls-1" d="M242.18,26.67a3.16,3.16,0,0,1,1.73-.56,1.67,1.67,0,0,1,1.32.53,2.51,2.51,0,0,1,.46,1.7,14.42,14.42,0,0,1-.34,2.35l-.06.35-.12.74a10.81,10.81,0,0,0-.25,1.86c0,.27.07.41.2.41a1.15,1.15,0,0,0,.72-.38l.19.4a1.54,1.54,0,0,1-.32.35,2.46,2.46,0,0,1-.61.39,1.76,1.76,0,0,1-.79.18c-.65,0-1-.31-1-.94a11.94,11.94,0,0,1,.28-1.87l.14-.74.06-.37c.1-.55.18-1,.24-1.45a8.14,8.14,0,0,0,.1-1.16,1.64,1.64,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.3,2.3,0,0,0-1.38.61,5.67,5.67,0,0,0-1.29,1.46l-.76,5.55h-1.45l1-7.44-.76-.36.13-.59,1.87-.34.32.17-.2,1.85a5.77,5.77,0,0,1,1.5-1.46"/><path class="cls-1" d="M252.77,37.29a4,4,0,0,1-1.55,1.05,5.36,5.36,0,0,1-1.78.32,5.69,5.69,0,0,1-1-.09,3.45,3.45,0,0,1-.76-.19l.32-1.24a5,5,0,0,0,.9.34,3.56,3.56,0,0,0,1.06.17,2.22,2.22,0,0,0,1.45-.39,2.12,2.12,0,0,0,.63-1.41l.29-2a3.73,3.73,0,0,1-1,.84A2.61,2.61,0,0,1,250,35a2.16,2.16,0,0,1-1.89-.88,3.9,3.9,0,0,1-.64-2.35,6.52,6.52,0,0,1,.58-2.6A5.62,5.62,0,0,1,249.73,27a4.15,4.15,0,0,1,2.61-.87,4.47,4.47,0,0,1,.87.1,5.15,5.15,0,0,1,.8.24l1-.34-1.38,9.31a3.37,3.37,0,0,1-.81,1.87m-.61-10.36a2.36,2.36,0,0,0-1.75.74,4.82,4.82,0,0,0-1.07,1.85,7.16,7.16,0,0,0-.35,2.15,3.3,3.3,0,0,0,.37,1.72,1.16,1.16,0,0,0,1,.62,2.09,2.09,0,0,0,1.2-.39,3.12,3.12,0,0,0,.9-.87l.82-5.5a1.85,1.85,0,0,0-1.15-.32"/><path class="cls-1" d="M258.39,33.67a.45.45,0,0,0,.07.28.19.19,0,0,0,.17.1.6.6,0,0,0,.35-.11,3.13,3.13,0,0,0,.44-.35l.2.37a3.17,3.17,0,0,1-.69.68,1.77,1.77,0,0,1-1.08.35,1.14,1.14,0,0,1-.78-.24.82.82,0,0,1-.22-.71c0-.26.16-1.25.4-3,.15-.93.32-2.17.52-3.7l-.95-.36.09-.52,2.19-.35.33.17-1,7.21Zm-.07-9.46a.81.81,0,0,1-.24-.6,1.09,1.09,0,0,1,.3-.78.94.94,0,0,1,.72-.33.81.81,0,0,1,.61.24.82.82,0,0,1,.23.58,1.17,1.17,0,0,1-.29.8,1,1,0,0,1-.77.33.76.76,0,0,1-.56-.24"/><path class="cls-1" d="M265.74,26.67a3.11,3.11,0,0,1,1.72-.56,1.65,1.65,0,0,1,1.32.53,2.5,2.5,0,0,1,.47,1.7,15.68,15.68,0,0,1-.34,2.35l-.06.35-.13.74a11.6,11.6,0,0,0-.24,1.86c0,.27.06.41.2.41a1.2,1.2,0,0,0,.72-.38l.18.4a1.54,1.54,0,0,1-.32.35,2.28,2.28,0,0,1-.61.39,1.73,1.73,0,0,1-.79.18.84.84,0,0,1-1-.94,12.8,12.8,0,0,1,.27-1.87l.14-.74.06-.37c.1-.55.19-1,.25-1.45a8.07,8.07,0,0,0,.09-1.16,1.64,1.64,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.27,2.27,0,0,0-1.37.61,5.7,5.7,0,0,0-1.3,1.46l-.75,5.55h-1.46l1-7.44-.75-.36.12-.59,1.88-.34.32.17-.2,1.85a5.77,5.77,0,0,1,1.5-1.46"/><path class="cls-1" d="M276.11,34.48a3.7,3.7,0,0,1-1.95.52,3,3,0,0,1-1.71-.46,2.87,2.87,0,0,1-1-1.23,4.24,4.24,0,0,1-.32-1.64,6.53,6.53,0,0,1,.59-2.75,5.17,5.17,0,0,1,1.63-2,3.76,3.76,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.53,1.76,1.76,0,0,1,.55,1.36,2.31,2.31,0,0,1-.88,1.84A5.83,5.83,0,0,1,274.87,31a9.08,9.08,0,0,1-2.15.4,3.49,3.49,0,0,0,.36,1.86,1.48,1.48,0,0,0,1.4.79,3.16,3.16,0,0,0,1.35-.31,3.9,3.9,0,0,0,1.22-.9l.33.57a3.93,3.93,0,0,1-1.27,1.1m-2.1-7a4.12,4.12,0,0,0-.88,1.44,6.37,6.37,0,0,0-.38,1.7,5.17,5.17,0,0,0,1.62-.34,3.78,3.78,0,0,0,1.4-.87,1.74,1.74,0,0,0,.57-1.28,1.31,1.31,0,0,0-.28-.91,1,1,0,0,0-.78-.32,1.68,1.68,0,0,0-1.27.58"/><path class="cls-1" d="M284,34.48a3.7,3.7,0,0,1-2,.52,3,3,0,0,1-1.71-.46,2.87,2.87,0,0,1-1-1.23,4.24,4.24,0,0,1-.32-1.64,6.53,6.53,0,0,1,.59-2.75,5.26,5.26,0,0,1,1.63-2,3.79,3.79,0,0,1,2.3-.77,2.31,2.31,0,0,1,1.64.53,1.76,1.76,0,0,1,.55,1.36,2.31,2.31,0,0,1-.88,1.84A5.74,5.74,0,0,1,282.73,31a9.08,9.08,0,0,1-2.15.4,3.4,3.4,0,0,0,.37,1.86,1.45,1.45,0,0,0,1.39.79,3.2,3.2,0,0,0,1.36-.31,3.86,3.86,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.28,1.1m-2.1-7a4.12,4.12,0,0,0-.88,1.44,6.37,6.37,0,0,0-.38,1.7,5.17,5.17,0,0,0,1.62-.34,3.7,3.7,0,0,0,1.4-.87,1.74,1.74,0,0,0,.57-1.28,1.31,1.31,0,0,0-.28-.91,1,1,0,0,0-.78-.32,1.7,1.7,0,0,0-1.27.58"/><path class="cls-1" d="M290.05,27.3a3.77,3.77,0,0,1,1-.83,2.34,2.34,0,0,1,1.22-.34.83.83,0,0,1,.5.11l-.33,1.62a1.45,1.45,0,0,0-.75-.15,2.3,2.3,0,0,0-1.29.44,2.89,2.89,0,0,0-1,1.22l-.7,5.47h-1.48l1-7.41-.77-.38.11-.57,1.81-.35.31.18-.11,1.37-.08.61a4.09,4.09,0,0,1,.6-1"/><path class="cls-1" d="M295.29,33.67a.45.45,0,0,0,.07.28.19.19,0,0,0,.17.1.6.6,0,0,0,.35-.11,3.13,3.13,0,0,0,.44-.35l.2.37a3.17,3.17,0,0,1-.69.68,1.77,1.77,0,0,1-1.08.35,1.14,1.14,0,0,1-.78-.24.82.82,0,0,1-.22-.71c0-.26.16-1.25.4-3,.15-.93.32-2.17.52-3.7l-.95-.36.09-.52,2.19-.35.33.17-1,7.21Zm-.07-9.46a.81.81,0,0,1-.24-.6,1.09,1.09,0,0,1,.3-.78.94.94,0,0,1,.72-.33.81.81,0,0,1,.61.24.82.82,0,0,1,.23.58,1.17,1.17,0,0,1-.29.8,1,1,0,0,1-.77.33.76.76,0,0,1-.56-.24"/><path class="cls-1" d="M302.33,26.67a3.14,3.14,0,0,1,1.73-.56,1.67,1.67,0,0,1,1.32.53,2.56,2.56,0,0,1,.46,1.7,15.68,15.68,0,0,1-.34,2.35l-.06.35-.12.74a10.81,10.81,0,0,0-.25,1.86c0,.27.07.41.2.41a1.2,1.2,0,0,0,.72-.38l.19.4a1.64,1.64,0,0,1-.33.35,2.22,2.22,0,0,1-.6.39,1.76,1.76,0,0,1-.79.18c-.65,0-1-.31-1-.94a12.8,12.8,0,0,1,.27-1.87l.14-.74.06-.37c.11-.55.19-1,.25-1.45a8.07,8.07,0,0,0,.09-1.16,1.73,1.73,0,0,0-.2-1,.88.88,0,0,0-.73-.27,2.3,2.3,0,0,0-1.38.61,5.7,5.7,0,0,0-1.3,1.46l-.75,5.55h-1.46l1-7.44-.75-.36.13-.59,1.87-.34.32.17-.2,1.85a5.77,5.77,0,0,1,1.5-1.46"/><path class="cls-1" d="M312.92,37.29a4,4,0,0,1-1.55,1.05,5.36,5.36,0,0,1-1.78.32,5.69,5.69,0,0,1-1-.09,3.57,3.57,0,0,1-.77-.19l.33-1.24a5,5,0,0,0,.9.34,3.51,3.51,0,0,0,1.06.17,2.22,2.22,0,0,0,1.45-.39,2.11,2.11,0,0,0,.62-1.41l.3-2a3.73,3.73,0,0,1-1,.84,2.63,2.63,0,0,1-1.33.34,2.16,2.16,0,0,1-1.89-.88,3.9,3.9,0,0,1-.64-2.35,6.35,6.35,0,0,1,.58-2.6A5.62,5.62,0,0,1,309.88,27a4.11,4.11,0,0,1,2.61-.87,4.56,4.56,0,0,1,.87.1,5.15,5.15,0,0,1,.8.24l.95-.34-1.38,9.31a3.37,3.37,0,0,1-.81,1.87m-.62-10.36a2.35,2.35,0,0,0-1.74.74,4.7,4.7,0,0,0-1.07,1.85,7.16,7.16,0,0,0-.35,2.15,3.3,3.3,0,0,0,.37,1.72,1.14,1.14,0,0,0,1,.62,2,2,0,0,0,1.19-.39,3.3,3.3,0,0,0,.91-.87l.82-5.5a1.88,1.88,0,0,0-1.16-.32"/><path class="cls-1" d="M237.11,49.24c.83-1.82,1.39-3.07,1.65-3.76h3.29l-.06.66-1.37.17-.29,9.8,1.31.18-.07.6h-4.22l.06-.6,1.4-.18.35-6.79.2-3.12L238.2,49l-3.57,8-.83.11-1.26-7.89-.47-2.92-.63,3-1.64,6.91,1.55.18-.06.6h-4l.06-.6,1.28-.18,2.47-9.8-1.41-.17.09-.66h3.56L234.56,53l.2,1.57.67-1.53c.28-.69.84-1.95,1.68-3.77"/><path class="cls-1" d="M247.75,56.54a3.75,3.75,0,0,1-1.95.52,3,3,0,0,1-1.71-.47,2.7,2.7,0,0,1-1-1.23,4.19,4.19,0,0,1-.32-1.63,6.35,6.35,0,0,1,.59-2.75,5.17,5.17,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.52,1.82,1.82,0,0,1,.55,1.37,2.34,2.34,0,0,1-.89,1.84A5.79,5.79,0,0,1,246.5,53a8,8,0,0,1-2.15.4,3.51,3.51,0,0,0,.37,1.87,1.48,1.48,0,0,0,1.4.79,3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.27,1.11m-2.11-7a4.25,4.25,0,0,0-.87,1.45,6.25,6.25,0,0,0-.39,1.69,5.44,5.44,0,0,0,1.63-.33,4.05,4.05,0,0,0,1.4-.87A1.76,1.76,0,0,0,248,50.2a1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/><path class="cls-1" d="M257.17,55.72q0,.39.24.39a1.19,1.19,0,0,0,.7-.39l.22.39a3.42,3.42,0,0,1-.67.62,1.68,1.68,0,0,1-1,.31c-.6,0-.9-.32-.9-1v-.34a3.94,3.94,0,0,1-1.08.94,2.72,2.72,0,0,1-1.44.38,2.13,2.13,0,0,1-1.87-.88,4,4,0,0,1-.63-2.34,6.41,6.41,0,0,1,.58-2.62A5.56,5.56,0,0,1,253,49a4.1,4.1,0,0,1,2.61-.86,4,4,0,0,1,1.15.18l.41-2.76-1-.26.13-.54,2.16-.28.34.2-1.6,10.84a.79.79,0,0,0,0,.17M255.42,49a2.32,2.32,0,0,0-1.74.75,4.64,4.64,0,0,0-1.06,1.84,7.15,7.15,0,0,0-.34,2.16,3.44,3.44,0,0,0,.36,1.72,1.12,1.12,0,0,0,1,.61,2.06,2.06,0,0,0,1.23-.41,3.11,3.11,0,0,0,.9-.91l.83-5.42a1.94,1.94,0,0,0-1.2-.34"/><path class="cls-1" d="M262.14,55.72a.57.57,0,0,0,.06.29.24.24,0,0,0,.18.1.71.71,0,0,0,.35-.11,3.84,3.84,0,0,0,.44-.35l.19.37a2.88,2.88,0,0,1-.68.67,1.71,1.71,0,0,1-1.08.35,1.19,1.19,0,0,1-.78-.23.8.8,0,0,1-.22-.72c0-.25.15-1.25.4-3,.14-.93.32-2.16.52-3.7l-.95-.35.09-.52,2.18-.36.34.17-1,7.22Zm-.07-9.46a.81.81,0,0,1-.24-.6,1.12,1.12,0,0,1,.3-.78,1,1,0,0,1,.71-.32.83.83,0,0,1,.62.24.81.81,0,0,1,.23.57,1.2,1.2,0,0,1-.29.81,1,1,0,0,1-.77.33.73.73,0,0,1-.56-.25"/><path class="cls-1" d="M269.42,49.14a2,2,0,0,0-1.35.54,3.6,3.6,0,0,0-.92,1.52,7,7,0,0,0-.33,2.24,3.52,3.52,0,0,0,.43,2,1.39,1.39,0,0,0,1.24.66,2.38,2.38,0,0,0,1.1-.24,5.05,5.05,0,0,0,1-.72l.3.57a4,4,0,0,1-1.14,1A3.43,3.43,0,0,1,268,57a2.44,2.44,0,0,1-2-.91,4,4,0,0,1-.73-2.54,6.43,6.43,0,0,1,.53-2.51A5.3,5.3,0,0,1,267.33,49a3.69,3.69,0,0,1,2.42-.83,5,5,0,0,1,.93.09,3.09,3.09,0,0,1,.81.26L271.07,50l-.26-.09a2.53,2.53,0,0,0-1.39-.78"/><path class="cls-1" d="M274.77,55.72a.56.56,0,0,0,.07.29.24.24,0,0,0,.18.1.71.71,0,0,0,.35-.11,3.15,3.15,0,0,0,.43-.35l.2.37a2.88,2.88,0,0,1-.68.67,1.71,1.71,0,0,1-1.08.35,1.19,1.19,0,0,1-.78-.23.8.8,0,0,1-.22-.72c0-.25.15-1.25.4-3,.14-.93.32-2.16.52-3.7l-1-.35.09-.52,2.18-.36.34.17-1,7.22Zm-.07-9.46a.85.85,0,0,1-.23-.6,1.12,1.12,0,0,1,.3-.78.94.94,0,0,1,.71-.32.79.79,0,0,1,.61.24.78.78,0,0,1,.24.57,1.16,1.16,0,0,1-.3.81,1,1,0,0,1-.76.33.75.75,0,0,1-.57-.25"/><path class="cls-1" d="M282,48.73a3.09,3.09,0,0,1,1.73-.56,1.65,1.65,0,0,1,1.32.52,2.54,2.54,0,0,1,.46,1.7,15.09,15.09,0,0,1-.33,2.35l-.07.36-.12.73a11.6,11.6,0,0,0-.24,1.86c0,.28.06.42.19.42a1.22,1.22,0,0,0,.73-.39l.18.4a1.77,1.77,0,0,1-.32.36,3.21,3.21,0,0,1-.61.39,1.86,1.86,0,0,1-.79.17c-.64,0-1-.31-1-.93a13.42,13.42,0,0,1,.28-1.88l.14-.73.06-.37c.1-.56.18-1,.25-1.45a8.17,8.17,0,0,0,.09-1.16,1.67,1.67,0,0,0-.21-1,.87.87,0,0,0-.73-.27,2.31,2.31,0,0,0-1.37.61,5.58,5.58,0,0,0-1.3,1.47l-.75,5.54H278.2l1-7.43-.75-.37.12-.58,1.87-.34.33.17-.2,1.84A5.85,5.85,0,0,1,282,48.73"/><path class="cls-1" d="M292.42,56.54a3.75,3.75,0,0,1-1.95.52,3,3,0,0,1-1.71-.47,2.7,2.7,0,0,1-1-1.23,4.19,4.19,0,0,1-.32-1.63A6.35,6.35,0,0,1,288,51a5.17,5.17,0,0,1,1.63-2,3.7,3.7,0,0,1,2.29-.77,2.32,2.32,0,0,1,1.65.52,1.82,1.82,0,0,1,.55,1.37,2.34,2.34,0,0,1-.89,1.84A5.79,5.79,0,0,1,291.17,53a8,8,0,0,1-2.15.4,3.51,3.51,0,0,0,.37,1.87,1.48,1.48,0,0,0,1.4.79,3.18,3.18,0,0,0,1.35-.32,3.75,3.75,0,0,0,1.21-.9l.34.57a4,4,0,0,1-1.27,1.11m-2.11-7a4.25,4.25,0,0,0-.87,1.45,6.25,6.25,0,0,0-.39,1.69,5.44,5.44,0,0,0,1.63-.33,4.05,4.05,0,0,0,1.4-.87,1.76,1.76,0,0,0,.57-1.28,1.33,1.33,0,0,0-.29-.92,1,1,0,0,0-.77-.31,1.74,1.74,0,0,0-1.28.57"/></g></g></svg> </a> </div> <div><a href="http://www.nasonline.org/" rel="noreferrer" target="_blank" class="global-footer__org"><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAuCAYAAABTTPsKAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAbkSURBVHgBrVlNkBNFFH49k+wui8sGXORHChLZyHKgCBzkolW7anldAqVHgbNacPTn4O5FOYJ/Fw+AnigLA1dL3bU84QFS5YGfXdxAYSF/svwssMmmx/d10nHS6ZnMJHxVyfx0z5uvX7/3+vUbQV3ifCGdem4pkXMcN0eO3MS3UkQiJchLeUQlKcVVPi/JKhVffmemSF1CUAcAyRWyd79w5LhHIlcjGQnz/Dstl+SJ7Nuz09QBYhEG0RT1HJREhyg6ySCUPOlNDu+dOR7nociE585sOSilN0HdEzVRKlN5bGu+VIrSuS1haHWQkgWe+tF2fRP9L9ILOw9Tz+BWWnp8nf45+x4f/6YocIgmMvnLkxH6BeNCYSS9gnrOg6yTXKEIBQridhDtG3qFzweowkTX7Po69Bk/2Mwm/ipkp6CgsH6BGr5cyOZcElN8mgKJtbu+oetT400aA8nBzfsUQWDZ0C5FUFYeKMK9gyN098/P1CDwqqd3ztLCjV9UewhCTcRKGJrtIQmyaVyvf/V79dLrU7upfP+CIgWiePHDawU1CJDf+NavfP2jau9lbQPXfnq9IXcZy9Dk7185EULcK/KQx3bkS/Nmi9uOrCYHQv1rXlN9+lZtp/mZb+nJrd/5pQ9rr5CL9Jiv0ReEQBiknt75oyEbA8M1jkO5Saou3gmwcbG2l9yRL07ePWm2tNgwk/1Ukx3YmFdaA1kQ09N97+JXLdpB2+qdnyui0OTtcx8y4e/IBpC8yQ6JfitHPqAA7L5yasuhlqH4L2ZPZXkxEMf0NbSlp3eRTQHmABPwY/m6N+n5bR9bnQvE8KsozZ5lGRdbBorn0Q7Z/ufqmGd73uG35wZh0xQ0oN11bMO8vCobNrFh7AxHhxHqBgiBif4N6rxcV8wjds7HN37Ge6dfys+M6b4JfdLjSPaiZrIAPN3lKLDo04B/MO3IaudKWmYAGl9gUtAoNA1ZZb6Ha/0+hNSZH4ZH9VKuCM9x7JOS9psCMc39LAhHfY5RayQjxFj0f+JzvCCAOOTD7k2zcRMCfjWNc+V0VZnEXKdNIXCImoPVIsEQ2yocUUNr16b9TnDv4pfsN++23IeWsS40CPP/PrMT4iUcBaOFx4PUE75ezUsvIgeO2sP9WvcDUxtFu/7+MDP8TLhSKAdy4GzCkifApnREqGLlYsK3z32kyAPQNMwEgwoCNBYXMA3/LDbg0DgOiWS1kiO3ef2ojXLAKhCDwA9CEUdhc8gZnhWwsCxTq2ELcsgzHHKcUbMFDzzyxVvbagTSt1jjsOOgBMc2tVEQZBYD1DPqcKDbbjbAfhFyoiBkpQqcpXaAlvssWvakSDuCnLTtoTYZlQKEhqWPyYippQk4uO1Zx5HbOUp46XYCdFgzAccMg+jQJJbqqakJzxMrnSgCgrSN/DcMOsXsBFWLkhzhDVoJRzGHKMtyokOTCH0v1bbeTbDZDyKCHwGhp/UFHZpFoDy2tBbC1QCb9aMvMuH4kULn3SZQmIHTFe0vCtdMO/vV6CRSgLAtrKKK5Hied9VsQD4aZp8QGDUH7sSOYW5lS0IlHCo60qMWDWNnGxayuvH+KEAyb1tdXSoXnaRTOW02wH7CNIMtUVTEdbqQhKqY4a2Sw3/zHm9DzFbsFGxZE5biONMc1+mwj7SlpGy6vyl5+OPl+YzZobaeNzsWiK4ceZ/iII6GdXSwmoOoHFHyan+Lx8kSj1EU8RMMS3SC4MbQMOTbSgPYiGbqO2dFGGbBta2jZkdo2V8zsybWzwggi52LLf5Wl7xGkbCxNCeoDJW3aBm1seXr3qBVHWg3KgY27iEsCwuWrRa06y9+NwhnVB1LWsudSNZFfWrhwTYbQ0an6glsRmaxpR3ZRP96tdm1QVDlgP864b/YnJ89MlvIjpt7PHiujr3/smBtKr313QZqCf6dM+7BfOADDwLKVVouokgQWa6TTGb2NlcxE2Yfl9wD0qgA6WII7Aw/aBL32u2IzYH4BwQ5mI1gGeL05r2XJlru2rrO8U6aSZ8n4/MASlYAds7QILQM4qaJoASAdpC96avCw3GxgmLwtoKJD5zklHdkLOXWwIL2HBcuZL2gre/VKpSHlYbhjLoUqwFtgcTQtk+acg30X1AR4KGy73YFbSY7lolT0P6ftNK0tUBovtRv07YqJci22xhgxXWpkrdpNhJhTbpK1WMiwkeZbsBL79HhPTOH2vWL/NnrSmGYhTkHyVKD6xIlnsUD2Xy0D42xPiwqbXtyQghVNur2ex2vrmIym790JM5DHX26rZvJKJ/ui2kqsM1p1ujRqBo10RFhP1BbXiL+OE5uzvNkWgixiZ0nVRde8jy6L4TD014tdkrSj/8AsRRWhMxQo+MAAAAASUVORK5CYII=" alt="NAS Logo" width="17" height="17"> <span> National Academy of Sciences </span></a> <a href="https://www.nae.edu/" rel="noreferrer" target="_blank" class="global-footer__org"><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAYAAAAehFoBAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAOgSURBVHgB1VlbTttAFD0zTgrqg6YrqPnjJZGuoGEFhRUQVtDkr6p4OFBV/WtYQZMVACsgOyCVKOLT7CBCtGoh9vRe23k4sR0nIcQ5UoIfM+bkzJ17z3gExoVeyGD+ZRaayNLZW0Bm+Kp31wTsBv29gaXquP5Sw5gQGAVM8sWrPB19oEfkYvcTaMBWNTo6w9VRBSNgOMJLn3Ro6QMIsQmFDMaDSc+owL6v4vqbGbdTPMKs6POFj9TawOODiKtSXMUHE175nIVInaATl5OBUqewH4qD1I4mvLq/TS3KjzD8cWHCut+IIi1Du67tH9B35QnJMmiOPLtwRjUEwQozWTWReI2LBlRzA1df6703+gkv7eagyXNMH4Hh4Q8JJ23JH0gGiMtcHxc/Yc6xk84GQ0HlsLxX6L7SCYnl3U1IeYLkoYH07SLqZS7xXQpL+R3JRAb3C22VXYXXaKKpREy0MLRVdhVWYhvJRgb/HLOFlHNqaSX6LiHJ+IsGZhECq5Q2FN6TTz0ONNgre3lqRSEjdPqQs7Krgc6KC44U5OjYyFM7m8zM3G21Nbt97TRt20lZHJsgY289lPr8Q8chbjrnSplsjojwwbnXGU45vjrshMbqHiVuke8jB1XGr6Ni+zS8lPurVXjqJCGaW+1S7CwQFi7QWxNsVaRJpzrmhv0uK8BgZQPJOg0L7n241THcd/irVXjq1D0L68Jdzej9/1auy74b0hsCQcMbBSHdzCLTuch2PHpZUmzNEUKPaKi3xUJY1lJvmLDfPgpt3TsKtXheZ93rMKAd4XdGhy30ge1ku03wM4V4He6H4yKOX9aaGXfSxsCSEdmOCc9UfpstwgqmpPxWx6xA2T9JYXGDWQGJKyHtU8wK5u+IcCrFITEDcSxqrr2sG0RWJD+O2cOgteIQVrKtJUeArdX4wCV8yS5N1JBYiFNcGyYfdSpdclUmxyfa3DqEE6sy+3RXXYbfS1hih4I7SRmjTr673H3BT5h/yR9tB8kAhYLc6r3Y79ZMY/oK89aCIrJdodCCjOg0vUloS3pzaQTWhlRop8tDg9Z0XFSe9o1QgKrdiDbwHPBKvoOzfTVp0O6SVxyiMHjFwUNj0RAJHGMSEM5Sv0jibAxSlxFvicQPujwsEPFFOqviMcBEeZ7c0TN7UlcUUhgGrgJ5WncZkFYOw8KZ/c6C4Qx3WmWUjDTaTmg3eNGYbmZJfdoeE7R127WUdwjSh0tr693YmGnzPxgRRC0B+OQMAAAAAElFTkSuQmCC" alt="NAE Logo" width="17" height="17"> <span> National Academy of Engineering </span></a> <a href="https://nam.edu/" rel="noreferrer" target="_blank" class="global-footer__org"><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC4AAAAuCAYAAABXuSs3AAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAiySURBVHgBrVl5bFRFGP/eUmppjd0GWs+yD1GpVOy2CWCkSV8xAvHq4oFY0S71iqhQxeg/2sPEoP3DtlIPiPZQolESbaPEqNguKh6gsBUqNKjdVuUqR6vQgNA+5zdv3/Ydsyf9JS+znTcz7zfffNd8lWg84PU46SwpJJFMDnIROWQi1am9lAZJHe0ilVhLftrQ5qNxgBTX6FKPe2F+gbJgltuVdl4KJ7a5u0tmjzI4fDKelXw0qrbSBGqjlrZBSgDRiS/zKBU33Voy46JLvUtmX+90pqYJh23p6abWbzupdauP4gAjLbWRNFrDNhCIZ2JY4sqa55QVxYuqbsi9VglHVoTWrZ301PvNNDg8THFBlarJMdoQ6wnYiXs98qPzFzU/vbBEkadkUSLo6g/Q/NrK+MkTBUhSi2ORvsPWc5bkxnsfSpg0kDdVpqbyxykByEzyvXSfpyLaQDvxJPLvCPyRkMEYUVIwh4pm5EYdBwHZhOSQ6uj+26sizZtg6/HvPbXjguSe5OSkpe6p0yhWwKukTEy2kRIZK9ZdOreQFs3KJyXnGt6WFMzlv0+f+Y8CRwagxArlzWR6t2eL6Hti4yxbvBPry1MyKS97GhXlzCTX5AvJmZrKX/cdHWCLH6Y+9vh6fuUtUFmyhFbdeEtoHDCh/E7T0hULbuFzV7EWREUCwFPT9gG1MC/F3OaT9G5bfXTiOCJJraYEUFao0CtLy03EL39mRWhj3sJi3taVYkx0T1XNyGMDQYP1Gd+ZdZz57ERJQy1ck7NMpCE5nTTe4+8qz90Uq3utZmPrSpfDVTbz6ByW+AQ2IEHcP08hqJYRCEo6lJxcbrDxequKBbdis8zbOEzGmhT6dZ/HS3BHCUCXZtm8Jab+hi828RYSxuMtnG+bCz1u/baDGyQ23vzgE7bNQfJDwycr6iUeoALoG5O4Q4rofiKRvi1/NjPKm039bTu2hSTuZn5dlMtAf5e/tZZ8e7u5waLNr1zN5v5oGw8Vu8w5OeTfNXcIaUuSlxIgXTRjJndn102/KtQPEqXr6piUtMgJ74GxRi/i27ubkW60rXnqzBn68fd9dHF6BhndMVzt2ZGRnM3Jp9fBZWsSd0hllCDpIuhu/hzTO+QqffDFQUB61uNHQhYO2DikjhMw4uHiBc4rplzs5ZSRm7BWoQRJl80rNr2raf+Q2nduN/VBv10W4n6Wz0QCNruFnYp1nacW3VaiET9LboqTNHS6jPlkEekX2BMLjkfJ3yFtkA8E3amOOdOvdMM1JjHqCsVJGoEkL1s2vWv48tOIpCE9PdpCckPDJygaML6dGTmirI7pWRfBn7uTmFHmUQyIRFrLwVtsc2BcukdRSTt+XWWqPUu51LUNDdgkC2CD/v5eW9/C3AJGnFQ52kUoEmkcafnbr5kW9rBAA6L5jHh68O9w0dLfrwWl+i8+ZX7/E1MOL1IVYGFunsyIS06KgnCku/4M0B2NL4f+xhiQSGdhH8EmltCuuzwEGcwvfqkyRBbE0wVrjEijMtxhROIrWWDJYJOtpLH44rUvcwmBbEVQD2G0CNPxXPd0YJ2PVz5ryncCBreq4+Sp0+lJkRZCSgsCSFeNgCTm11ZxXw2JIQ+B7tYZjMgIBBvfnt18Leg1iDlTz2d20cTIXsjnwwBBHOth48gMgXACAHHcdoRSd091MZen2PrL327kpKG7AKQsunQ0ML0FAS2PYSnvPeaUF54IxocHOg51QWhHfNAhcWsxIy3lvCGmKqrwmoZrl0twrdIDDPQRhJBbW0mDCHKOiveaQjkHdNVqaNZsEpuEsevfRCuS+P5jxwPQcT8JgAuvVdr4MHw1iIKQKJODlEHa6sb6jh5mxmzuS59kJ4VM0ZhgiU6S5SyMuEpdJAAM0koKpCEB+GZRbo1sD1IWQSQ5UW6uBakxg4QArVjbscnvYJHTZ32Rx3Z5bbZ9QvvObSG9tubWIK0blAiBAZAxxwujvo/1pXFjRgvJW++lPi1/8euqYtJz56RU20ege/AEAAwoHtLhoK9nBIgil9cEJNlOZc/ff/lQ7XLwkpfKCpAGyJmZlJFmlgb0Uzcm42I42kRIh4OLfUM34pKC2bb3nT27ONdgPs6qplGgB5oygYrEAggjfVJqxDHw50iq0IK8VR2hJhu3f+cbI65d/X36AH9/HzsE8eJug7FAB1siXAiMwNiMtDRLnzlDRDzAevAkZYXFtjX2HTzQYr9zSmqN/hOpp9XHQtqBYEqqw+ryIoEXliwlOWs4x1UOug11sUob33649fUQxzHiBqlr9RDzoqI64FCM1VhsVhHM77LcgvQ6oqga8MyHrQ3GKq65riKpyynoYXw93ZZFM22JVqzAPJflBPnVzPANjXAxz1msPv+9778OMN2uNvaZiWNHEqvVMbz65SZbfRuXYqMxGnOKSGh64DGbW0NMMJKu8iwJJVpGQEXuXV+32FrwF1Vr/ZSfI7EygZIycSK3cOMHuphef77bz/0tSgawh0gXX9Rb7maVWStub6zlqgadXnPXMtbOtVV7QXrFO+ue/K32jahebwxeT7Xz0WXq74cOqCMjI6ano/sXteqj99XegUPq8ZMnVHn1Iyqr8NqeVRvess3F0/T1ZlVZ87zauWeXGg5YW3nxuepw9KRo5Iuuuqaq49ka4WvcNaFOElulnpXbhnCnnJzJVUh0Y9Kl2H/siLDEbBwDSX+2urKeEobXU7EyjOTifXB6zd98pUbCxm1bj+MfZzQuYEWj0jfrekVqEw/pj3/+ISxhqNxj76zvDBaoxhd3vVrr3eT/uTcewkf//Uft+HWXurPvj7CEN277rnP8pBxlA09sWN/7FTNSEBORxbuPfvohLGEY5rkQju9f4lZ4PW4aJe/Vl2TnZV2QrkzLhGHO4vmM9doFg/vt0MHBfQf3+z/p2t7+2a4dbfH+N3n8iFuh6ac885JsOSstA79pUkrSICMZIOT950DUiv8BEppcI32BQ8oAAAAASUVORK5CYII=" alt="NAM Logo" width="17" height="17"> <span> National Academy of Medicine </span></a></div></div> <div class="global-footer__links"><ul class="global-footer__nav"><li><a rel="" title="" href="//www.nationalacademies.org/about" target="_self"> About Us </a></li><li><a rel="" title="" href="//www.nationalacademies.org/our-work" target="_self"> Our Work </a></li><li><a rel="" title="" href="//www.nationalacademies.org/events" target="_self"> Events </a></li><li><a rel="" title="" href="//www.nationalacademies.org/contact-us" target="_self"> Contact Us </a></li><li><a rel="" title="" href="//www.nationalacademies.org/opportunities" target="_self"> Opportunities </a></li><li><a rel="" title="" href="//www.nationalacademies.org/publications" target="_self"> Publications </a></li><li><a rel="" title="" href="//www.nationalacademies.org/about/visiting-us" target="_self"> Visit </a></li><li><a rel="" title="" href="//www.nationalacademies.org/about/operating-status" target="_self"> Current Operating Status </a></li><li><a rel="" title="" href="//www.nationalacademies.org/subscribe" target="_self"> E-Newsletters </a></li></ul> <div class="global-footer__social"><p> Connect With Us </p> <ul><li><a href="https://www.facebook.com/NationalAcademies/" target="_self" class="connect-facebook"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path fill-rule="evenodd" d="M20 9.375h-2.222c-.334 0-1.111.531-1.111 1.125v2.25H20v3.375h-3.333V24h-3.334v-7.875H10V12.75h3.333V10.5c0-2.482 2.18-4.5 4.515-4.5H20v3.375"></path></svg></a></li><li><a href="https://twitter.com/TheNASEM" target="_self" class="connect-twitter"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path fill-rule="evenodd" d="M24 9.775a7.238 7.238 0 01-2.121.598 3.787 3.787 0 001.625-2.096 7.253 7.253 0 01-2.347.92A3.65 3.65 0 0018.461 8c-2.04 0-3.695 1.697-3.695 3.787 0 .298.033.586.096.863-3.07-.157-5.793-1.665-7.615-3.956a3.853 3.853 0 00-.499 1.904c0 1.313.853 2.472 1.845 3.151-.606-.02-1.473-.191-1.473-.475v.05c0 1.833 1.072 3.363 2.761 3.712-.31.086-.736.133-1.072.133a3.99 3.99 0 01-.746-.07c.47 1.505 1.81 2.6 3.425 2.632A7.343 7.343 0 016 21.299 10.277 10.277 0 0011.66 23c6.795 0 10.51-5.77 10.51-10.774 0-.163-.005-.328-.013-.491A7.588 7.588 0 0024 9.775"></path></svg></a></li><li><a href="https://www.linkedin.com/company/the-national-academies" target="_self" class="connect-linkedin"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path fill-rule="evenodd" d="M7 23V10.533h3.4V23H7zm12.467-12.467c2.558 0 4.533 2.034 4.533 5.665V23h-3.4v-6.347c0-1.586-.779-2.679-2.192-2.679-1.082 0-1.94.736-2.223 1.443-.097.251-.118.608-.118.959V23h-3.4s.088-11.333.035-12.467h3.365l-.04 2.267c.516-1.133 1.335-2.267 3.44-2.267zM10.4 7.7a1.7 1.7 0 11-3.4 0 1.7 1.7 0 013.4 0z"></path></svg></a></li><li><a href="http://www.nationalacademies.org/news.xml" target="_self" title="Share our site" class="connect-rss"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path d="M11.85 20.537a2.425 2.425 0 11-4.85 0 2.425 2.425 0 014.85 0zm6.65 1.79c-.316-5.854-5.005-10.548-10.864-10.865a.605.605 0 00-.636.605v1.82c0 .32.246.586.564.607a8.484 8.484 0 017.904 7.904c.02.318.288.564.607.564h1.82c.346 0 .624-.29.605-.636zm5.462.01C23.644 13.484 16.52 6.32 7.625 6A.605.605 0 007 6.606v1.82c0 .327.259.593.585.606 7.238.297 13.048 6.108 13.345 13.345.013.326.28.585.606.585h1.82a.605.605 0 00.606-.625z"></path></svg></a></li><li><a href="//www.nationalacademies.org/subscribe" target="_self" title="Subscribe" class="connect-email"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path fill-rule="evenodd" d="M6 12.964v4.569c0 .296.197.386.439.2l2.822-2.15c.243-.184.243-.484 0-.669l-2.822-2.15c-.242-.185-.439-.095-.439.2zM7.125 9C6.506 9 6 9.601 6 10.071l8.514 6.455a.775.775 0 00.882.002L24 10.07C24 9.601 23.494 9 22.875 9H7.125zm13.613 6.584l2.822 2.15c.243.185.44.095.44-.201v-4.569c0-.295-.197-.385-.44-.2l-2.822 2.15c-.242.185-.242.485 0 .67zM18.26 16.8l-2.822 2.15a.764.764 0 01-.879 0l-2.822-2.15a.765.765 0 00-.88-.001L6 20.495v.29c0 .471.506 1.072 1.125 1.072h15.75c.619 0 1.125-.6 1.125-1.071v-.29L19.141 16.8a.767.767 0 00-.88.001z"></path></svg></a></li></ul></div> <div class="global-footer__legal"><div><button class="button button--icon button--reverse button--secondary"> <a href="https://www8.nationalacademies.org/AcademyGiving/nas/nasform.aspx ">Donate Today</a> <span class="button__icon"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30" class=""><path d="M17.885 10.708a1 1 0 111.412-1.416l5.624 5.608-5.616 5.732a1 1 0 11-1.428-1.4l4.228-4.316-4.22-4.208z"></path><path d="M23.07 14a1 1 0 010 2H7a1 1 0 010-2h16.07z"></path></svg></span></button></div> <div><ul><li><a rel="" title="" href="//www.nationalacademies.org/legal/privacy" target="_self"> Privacy Statement </a></li><li><a rel="" title="" href="//www.nationalacademies.org/legal/policy" target="_self"> DMCA Policy </a></li><li><a rel="" title="" href="//www.nationalacademies.org/legal/terms" target="_self"> Terms of Use </a></li><li><a rel="" title="" href="//www.nationalacademies.org/site-map" target="_self"> Site Map </a></li></ul> <p class="copyright"><a rel="" title="" href="//www.nationalacademies.org/legal" target="_self"> Copyright © <script type="text/javascript"> <!-- var d = new Date(); var n = d.getFullYear(); document.write(n) //--> </script> National Academy of Sciences. All Rights Reserved. </a></p></div></div></div></div></footer> <script src="//nap.nationalacademies.org/includes/js/jquery.dlmenu.js"></script> <script src="//nap.nationalacademies.org/includes/js/jquery.placeholder.js"></script> <!--[if lt IE 9]> <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script> <![endif]--> <script> // Save to MyNAP handler $('a.saveit').one('click', function(e) { e.preventDefault(); $.getJSON('/napsave.php', { type: 'shelf', r: $(this).data('recordid') }, function(response) { $(e.target) .toggleClass('saveit saveit_active') .attr('href', '/mynap.php') .text('Added to Bookshelf'); }); }); </script> <!-- OneTrust Cookies Consent Notice start for nap.nationalacademies.org --> <script src="https://cdn.cookielaw.org/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="5639f7b9-d1de-4e1d-9f2f-e86c3f24727b" ></script> <script type="text/javascript"> function OptanonWrapper() { } </script> <!-- OneTrust Cookies Consent Notice end for nap.nationalacademies.org --> <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.16.0/umd/popper.min.js"></script> <script src="/includes/js/vendor/bootstrap/4.0.0/bootstrap.bundle.min.js"></script> <script src="/includes/js/bootstrap.min.js"></script> <script> $("#searchagain").click(function() { $(".searchbook").slideToggle("slow", function() { }); }); </script> <!-- 144.171.19.8 --> </body> </html>