CINXE.COM

<!DOCTYPE html> <html class='v2 list-page' dir='ltr' itemscope='' itemtype='http://schema.org/Blog' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <title> Google Testing Blog: 2018 </title> <meta content='width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0' name='viewport'/> <meta content='IE=Edge' http-equiv='X-UA-Compatible'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='en_US' property='og:locale'/> <meta content='https://testing.googleblog.com/2018/' property='og:url'/> <meta content='Google Testing Blog' property='og:site_name'/>  <meta content='Google Testing Blog' property='og:title'/> <meta content='summary' name='twitter:card'/> <meta content='@googletesting' name='twitter:creator'/> <link href='https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic' rel='stylesheet' type='text/css'/> <link href='https://fonts.googleapis.com/icon?family=Material+Icons' rel='stylesheet'/> <script src='https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js' type='text/javascript'></script>  <style id='page-skin-1' type='text/css'></style> <style id='template-skin-1' type='text/css'></style>  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://testing.googleblog.com/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://testing.googleblog.com/2018/' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://testing.googleblog.com/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="Google Testing Blog - RSS" href="https://testing.googleblog.com/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://www.blogger.com/feeds/15045980/posts/default" />  <meta content='https://testing.googleblog.com/2018/' property='og:url'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='' property='og:description'/>  <base target='_self'/> <style> html { font-family: Roboto, sans-serif; -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; } body { padding: 0; /* This ensures that the scroll bar is always present, which is needed */ /* because content render happens after page load; otherwise the header */ /* would "bounce" in-between states. */ min-height: 150%; } h2 { font-size: 20px; } h1, h2, h3, h4, h5 { line-height: 2em; } html, h4, h5, h6 { font-size: 17px; } h3 { font-size: 18px !important; } a, a:visited { color: #4184F3; text-decoration: none; } a:focus, a:hover, a:active { text-decoration: none; } .Header { margin-top: 15px; } .Header h1 { font-size: 32px; font-weight: 300; line-height: 32px; height: 42px; } .header-inner .Header .titlewrapper { padding: 0; margin-top: 30px; } .header-inner .Header .descriptionwrapper { padding: 0; margin: 0; } .cols-wrapper { margin-top: 56px; } .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 60px; } .header-inner { height: 256px; position: relative; } html, .header-inner a { color: #212121; color: rgba(0,0,0,.87); } .header-inner .google-logo { display: inline-block; background-size: contain; z-index: 1; height: 46px; overflow: hidden; margin-top: 4px; margin-right: 8px; } .header-left { position: absolute; top: 50%; -webkit-transform: translateY(-50%); transform: translateY(-50%); margin-top: 12px; width: 100%; } .google-logo { margin-left: -4px; } #google-footer { position: relative; font-size: 13px; list-style: none; text-align: right; } #google-footer a { color: #444; } #google-footer ul { margin: 0; padding: 0; height: 144px; line-height: 144px; } #google-footer ul li { display: inline; } #google-footer ul li:before { color: #999; content: "\00b7"; font-weight: bold; margin: 5px; } #google-footer ul li:first-child:before { content: ''; } #google-footer .google-logo-dark { left: 0; margin-top: -16px; position: absolute; top: 50%; } /** Sitemap links. **/ .footer-inner-2 { font-size: 14px; padding-top: 42px; padding-bottom: 74px; } .footer-inner-2 .HTML h2 { color: #212121; color: rgba(0,0,0,.87); font-size: 14px; font-weight: 500; padding-left: 0; margin: 10px 0; } .footer-inner-2 .HTML ul { font-weight: normal; list-style: none; padding-left: 0; } .footer-inner-2 .HTML li { line-height: 24px; padding: 0; } .footer-inner-2 li a { color: rgba(65,132,243,.87); } /** Archive widget. **/ .BlogArchive { font-size: 13px; font-weight: normal; } .BlogArchive .widget-content { display: none; } .BlogArchive h2, .Label h2 { color: #4184F3; text-decoration: none; } .BlogArchive .hierarchy li { display: inline-block; } /* Specificity needed here to override widget CSS defaults. */ .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li { padding-left: 0; text-indent: 0; } .BlogArchive .intervalToggle { cursor: pointer; } .BlogArchive .expanded .intervalToggle .new-toggle { -ms-transform: rotate(180deg); transform: rotate(180deg); } .BlogArchive .new-toggle { float: right; padding-top: 3px; opacity: 0.87; } #ArchiveList { text-transform: uppercase; } #ArchiveList .expanded > ul:last-child { margin-bottom: 16px; } #ArchiveList .archivedate { width: 100%; } /* Months */ .BlogArchive .items { max-width: 150px; margin-left: -4px; } .BlogArchive .expanded .items { margin-bottom: 10px; overflow: hidden; } .BlogArchive .items > ul { float: left; height: 32px; } .BlogArchive .items a { padding: 0 4px; } .Label { font-size: 13px; font-weight: normal; } .sidebar-icon { display: inline-block; width: 24px; height: 24px; vertical-align: middle; margin-right: 12px; margin-top: -1px } .Label a { margin-right: 4px; } .Label .widget-content { display: none; } .FollowByEmail { font-size: 13px; font-weight: normal; } .FollowByEmail h2 { background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); background-repeat: no-repeat; background-position: 0 50%; text-indent: 30px; } .FollowByEmail .widget-content { display: none; } .searchBox input { border: 1px solid #eee; color: #212121; color: rgba(0,0,0,.87); font-size: 14px; padding: 8px 8px 8px 40px; width: 164px; font-family: Roboto, sans-serif; background: url("https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png") 8px center no-repeat; } .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */ color: rgba(0,0,0,.54); } .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */ color: #000; opacity: 0.54; } .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */ color: #000; opacity: 0.54; } .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */ color: #757575; } .widget-item-control { margin-top: 0px; } .section { margin: 0; padding: 0; } #sidebar-top { border: 1px solid #eee; } #sidebar-top > div { margin: 16px 0; } .widget ul { line-height: 1.6; } /*main post*/ .post { margin-bottom:30px; } #main .post .title { margin: 0; } #main .post .title a { color: #212121; color: rgba(0,0,0,.87); font-weight: normal; font-size: 24px; } #main .post .title a:hover { text-decoration:none; color:#4184F3; } .message, #main .post .post-header { margin: 0; padding: 0; } #main .post .post-header .caption, #main .post .post-header .labels-caption, #main .post .post-footer .caption, #main .post .post-footer .labels-caption { color: #444; font-weight: 500; } #main .tr-caption-container td { text-align: left; } #main .post .tr-caption { color: #757575; color: rgba(0,0,0,.54); display: block; max-width: 560px; padding-bottom: 20px; } #main .post .tr-caption-container { line-height: 24px; margin: -1px 0 0 0 !important; padding: 4px 0; text-align: left; } #main .post .post-header .published{ font-size:11px; font-weight:bold; } .post-header .publishdate { font-size: 17px; font-weight:normal; color: #757575; color: rgba(0,0,0,.54); } #main .post .post-footer{ font-size:12px; padding-bottom: 21px; } .label-footer { margin-bottom: 12px; margin-top: 12px; } .comment-img { margin-right: 16px; opacity: 0.54; vertical-align: middle; } #main .post .post-header .published { margin-bottom: 40px; margin-top: -2px; } .post .post-content { color: #212121; color: rgba(0,0,0,.87); font-size: 17px; margin: 25px 0 36px 0; line-height: 32px; word-wrap: break-word; } .post-body .post-content ul, .post-body .post-content ol { margin: 16px 0; padding: 0 48px; } .post-summary { display: none; } /* Capture section headers. */ .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b { display: inline-block; margin-bottom: 8px; margin-top: 24px; } .post-content li { line-height: 32px; } /* Override all post images/videos to left align. */ .post-content .separator, .post-content > div { text-align: left; } .post-content .separator > a, .post-content .separator > span { margin-left: 0 !important; } .post-content img { max-width: 100%; } .post-content .tr-caption-container img { margin-bottom: 12px; } .post-content iframe, .post-content embed { max-width: 100%; } .post-content .carousel-container { margin-bottom: 48px; } #main .post-content b { font-weight: 500; } /* These are the main paragraph spacing tweaks. */ #main .post-content br { content: ' '; display: block; padding: 4px; } .post-content .space { display: block; height: 8px; } .post-content iframe + .space, .post-content iframe + br { padding: 0 !important; } #main .post .jump-link { margin-bottom:10px; } .post-content img, .post-content iframe { margin: 30px 0 20px 0; } .post-content > img:first-child, .post-content > iframe:first-child { margin-top: 0; } .col-right .section { padding: 0 16px; } #aside { background:#fff; border:1px solid #eee; border-top: 0; } #aside .widget { margin:0; } #aside .widget h2, #ArchiveList .toggle + a.post-count-link { color: #212121; color: rgba(0,0,0,.87); font-weight: 400 !important; margin: 0; } #ArchiveList .toggle { float: right; } #ArchiveList .toggle .material-icons { padding-top: 4px; } #sidebar .tab { cursor: pointer; } #sidebar .tab .arrow { display: inline-block; float: right; } #sidebar .tab .icon { display: inline-block; vertical-align: top; height: 24px; width: 24px; margin-right: 13px; margin-left: -1px; margin-top: 1px; color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget-content > :first-child { padding-top: 8px; } #sidebar .active .tab .arrow { -ms-transform: rotate(180deg); transform: rotate(180deg); } #sidebar .arrow { color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget h2 { font-size: 14px; line-height: 24px; display: inline-block; } #sidebar .widget .BlogArchive { padding-bottom: 8px; } #sidebar .widget { border-bottom: 1px solid #eee; box-shadow: 0px 1px 0 white; margin-bottom: 0; padding: 14px 0; min-height: 20px; } #sidebar .widget:last-child { border-bottom: none; box-shadow: none; margin-bottom: 0; } #sidebar ul { margin: 0; padding: 0; } #sidebar ul li { list-style:none; padding:0; } #sidebar ul li a { line-height: 32px; } #sidebar .archive { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC"); height: 24px; line-height: 24px; padding-left: 30px; } #sidebar .labels { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC"); height: 20px; line-height: 20px; padding-left: 30px; } #sidebar .rss a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII="); } #sidebar .subscription a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); } #sidebar-bottom { background: #f5f5f5; border-top:1px solid #eee; } #sidebar-bottom .widget { border-bottom: 1px solid #e0e0e0; padding: 15px 0; text-align: center; } #sidebar-bottom > div:last-child { border-bottom: 0; } #sidebar-bottom .text { line-height: 20px; } /* Home, forward, and backward pagination. */ .blog-pager { border-top : 1px #e0e0e0 solid; padding-top: 10px; margin-top: 15px; text-align: right !important; } #blog-pager { margin-botom: 0; margin-top: -14px; padding: 16px 0 0 0; } #blog-pager a { display: inline-block; } .blog-pager i.disabled { opacity: 0.2 !important; } .blog-pager i { color: black; margin-left: 16px; opacity: 0.54; } .blog-pager i:hover, .blog-pager i:active { opacity: 0.87; } #blog-pager-older-link, #blog-pager-newer-link { float: none; } .gplus-profile { background-color: #fafafa; border: 1px solid #eee; overflow: hidden; width: 212px; } .gplus-profile-inner { margin-left: -1px; margin-top: -1px; } /* Sidebar follow buttons. */ .followgooglewrapper { padding: 12px 0 0 0; } .loading { visibility: hidden; } .detail-page .post-footer .cmt_iframe_holder { padding-top: 40px !important; } /** Desktop **/ @media (max-width: 900px) { .col-right { display: none; } .col-main { margin-right: 0; min-width: initial; } .footer-outer { display: none; } .cols-wrapper { min-width: initial; } .google-footer-outer { background-color: #f5f5f5; } } /** Tablet **/ @media (max-width: 712px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 40px; } } /* An extra breakpoint accommodating for long blog titles. */ @media (max-width: 600px) { .header-left { height: 100%; position: initial; top: inherit; margin-top: 0; -webkit-transform: initial; transform: initial; } .header-title { margin-top: 18px; } .header-inner { height: auto; margin-bottom: 32px; margin-top: 32px; } .header-desc { margin-top: 12px; } .header-inner .google-logo { height: 40px; margin-top: 3px; } .header-inner .google-logo img { height: 42px; } .header-title h2 { font-size: 32px; line-height: 40px; } } /** Mobile/small desktop window; also landscape. **/ @media (max-width: 480px), (max-height: 480px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 16px; } .cols-wrapper { margin-top: 0; } .post-header .publishdate, .post .post-content { font-size: 16px; } .post .post-content { line-height: 28px; margin-bottom: 30px; } .post { margin-top: 30px; } .byline-author { display: block; font-size: 12px; line-height: 24px; margin-top: 6px; } #main .post .title a { font-weight: 500; color: #4c4c4c; color: rgba(0,0,0,.70); } #main .post .post-header { padding-bottom: 12px; } #main .post .post-header .published { margin-bottom: -8px; margin-top: 3px; } .post .read-more { display: block; margin-top: 14px; } .post .tr-caption { font-size: 12px; } #main .post .title a { font-size: 20px; line-height: 30px; } .post-content iframe { /* iframe won't keep aspect ratio when scaled down. */ max-height: 240px; } .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe { margin-left: -16px; max-width: inherit; width: calc(100% + 32px); } .post-content table, .post-content td { width: 100%; } #blog-pager { margin: 0; padding: 16px 0; } /** List page tweaks. **/ .list-page .post-original { display: none; } .list-page .post-summary { display: block; } .list-page .comment-container { display: none; } .list-page #blog-pager { padding-top: 0; border: 0; margin-top: -8px; } .list-page .label-footer { display: none; } .list-page #main .post .post-footer { border-bottom: 1px solid #eee; margin: -16px 0 0 0; padding: 0 0 20px 0; } .list-page .post .share { display: none; } /** Detail page tweaks. **/ .detail-page .post-footer .cmt_iframe_holder { padding-top: 32px !important; } .detail-page .label-footer { margin-bottom: 0; } .detail-page #main .post .post-footer { padding-bottom: 0; } .detail-page #comments { display: none; } } [data-about-pullquote], [data-is-preview], [data-about-syndication] { display: none; } </style> <noscript> <style> .loading { visibility: visible }</style> </noscript>  <script async='true' src='https://www.googletagmanager.com/gtag/js?id=G-838ZCPQWM6'></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-838ZCPQWM6'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <script type='text/javascript'> //<![CDATA[ var axel = Math.random() + ""; var a = axel * 10000000000000; document.write('<iframe src="https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?" width="1" height="1" frameborder="0" style="display:none"></iframe>'); //]]> </script> <noscript> <img alt='' height='1' src='https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?' width='1'/> </noscript>  <div class='header-outer'> <div class='header-inner'> <div class='section' id='header'><div class='widget Header' data-version='1' id='Header1'> <div class='header-left'> <div class='header-title'> <a class='google-logo' href='https://testing.googleblog.com/'> <img height='50' src='https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png'/> </a> <a href='/.'> <h2> Testing Blog </h2> </a> </div> <div class='header-desc'> </div> </div> </div></div> </div> </div>  <div class='cols-wrapper loading'> <div class='col-main-wrapper'> <div class='col-main'> <div class='section' id='main'><div class='widget Blog' data-version='1' id='Blog1'> <div class='post' data-id='5108653381755768530' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/11/testing-on-toilet-exercise-service-call.html' itemprop='url' title='Testing on the Toilet: Exercise Service Call Contracts in Tests'> Testing on the Toilet: Exercise Service Call Contracts in Tests </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, November 27, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <span style="font-family: "times" , "times new roman" , serif;"><span style="font-style: italic; vertical-align: baseline; white-space: pre-wrap;">This article was adapted from a </span><a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Google Testing on the Toilet</span></a><span style="font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> (TotT) episode. You can download a </span><a href="https://docs.google.com/document/d/1MElViSQMCCl3_oL-_e5FEpul-6QKIUE6Ul5FqsOGnXA/edit?usp=sharing" rel="nofollow"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> of this TotT episode and post it in your office.</span></span><br /> <span style="font-family: "times" , "times new roman" , serif; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"><span style="font-style: normal;">By Ben Yu</span></span><br /> <br /> <span style="font-family: "times" , "times new roman" , serif;">The following test mocks out a service call to </span><span style="font-family: "courier new" , "courier" , monospace;">CloudService</span><span style="font-family: "times" , "times new roman" , serif;"><span style="font-family: "times" , "times new roman" , serif;">. </span><span style="color: purple; font-family: "times" , "times new roman" , serif;"> </span></span><span style="font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: "times" , "times new roman" , serif;">Does the test provide enough confidence that the service call is likely to work?</span></span><br /> <br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #cee2f3; vertical-align: top; width: 607px;"><pre style="background-color: #cee2f3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void uploadFileToCloudStorage() { when(<span style="font-weight: bold;">mockCloudService</span>.write( WriteRequest.newBuilder().setUserId(“<span style="font-weight: bold;">testuser</span>”).setFileType(“<span style="font-weight: bold;">plain/text</span>”)...)) .thenReturn(WriteResponse.newBuilder().setUploadId(“uploadId”).build()); CloudUploader <span style="font-weight: bold;">cloudUploader</span> = new CloudUploader(<span style="font-weight: bold;">mockCloudService</span>); Uri <span style="font-weight: bold;">uri</span> = <span style="font-weight: bold;">cloudUploader</span>.uploadFile(new File(“/path/to/foo.txt”)); // The uploaded file URI contains the user ID, file type, and upload ID. (Or does it?) assertThat(<span style="font-weight: bold;">uri</span>).isEqualTo(new Uri(“<span style="font-weight: bold;">/testuser/text/uploadId</span>.txt”));</span></pre> </td></tr> </tbody></table> </div> <br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">Lots of things can go wrong,</span></span><span style="font-family: times, times new roman, serif;"> especially when service contracts get complex. For example, </span></span><span style="font-family: "courier new" , "courier" , monospace;">plain/text</span><span style="font-family: "times" , "times new roman" , serif;"> may not be a valid file type, and you can’t verify that the URI of the uploaded file is correct.</span><br /> <br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">If the code under test relies on the contract of a service, prefer exercising the service call</span></span><span style="font-family: times, times new roman, serif;"> instead of mocking it out. This gives you more confidence that you are using the service correctly:</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void uploadFileToCloudStorage() { CloudUploader <span style="font-weight: bold;">cloudUploader</span> = new CloudUploader(<span style="font-weight: bold;">cloudService</span>); Uri <span style="font-weight: bold;">uri</span> = <span style="font-weight: bold;">cloudUploader</span>.uploadFile(”/path/to/foo.txt”); assertThat(<span style="font-weight: bold;">cloudService</span>.retrieveFile(<span style="font-weight: bold;">uri</span>)).isEqualTo(readContent(“/path/to/foo.txt")); }</span></pre> </td></tr> </tbody></table> </div> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span style="font-family: "times" , "times new roman" , serif;">How can you exercise the service call?</span><br /> <br /> <ol> <li><span style="font-family: "times" , "times new roman" , serif;"><span id="docs-internal-guid-d7ce8871-7fff-90fd-33fd-d97353a64fb8"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">Use a fake</span></span></span><span style="font-family: times, times new roman, serif;">.  A fake is a fast and lightweight implementation of the service that behaves just like the real implementation. A fake is usually maintained by the service owners; don’t create your own fake unless you can ensure its behavior will stay in sync with the real implementation.  Learn more about fakes at </span><a href="http://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html" style="font-family: times, "times new roman", serif;">testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html</a><span style="font-family: times, times new roman, serif;">.</span></span></li> <li><span style="font-family: "times" , "times new roman" , serif;"><span id="docs-internal-guid-ad45d921-7fff-e561-02d5-1a8cf098db47"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">Use a hermetic server</span></span></span><span style="font-family: times, times new roman, serif;">.  This is a real server that is brought up by the test and runs on the same machine that the test is running on. A downside of using a hermetic server is that starting it up and interacting with it can slow down tests.  Learn more about hermetic servers at </span><a href="http://testing.googleblog.com/2012/10/hermetic-servers.html" style="font-family: times, "times new roman", serif;">testing.googleblog.com/2012/10/hermetic-servers.html</a><span style="font-family: times, times new roman, serif;">.</span></span></li> </ol> <div> <span style="font-family: "times" , "times new roman" , serif;"><span style="font-family: times, times new roman, serif;">If the service you are using doesn’t have a fake or hermetic server, mocks may be the only tool at your disposal. But </span><span id="docs-internal-guid-648cf637-7fff-283e-ed04-f2bd572a8e54"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">if your tests are not exercising the service call contract, you must take extra care to ensure the service call works</span></span></span><span style="font-family: times, times new roman, serif;">, such as by having a comprehensive suite of end-to-end tests or resorting to manual QA (which can be inefficient and hard to scale).</span></span></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <span style="font-family: "times" , "times new roman" , serif;"><span style="font-style: italic; vertical-align: baseline; white-space: pre-wrap;">This article was adapted from a </span><a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Google Testing on the Toilet</span></a><span style="font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> (TotT) episode. You can download a </span><a href="https://docs.google.com/document/d/1MElViSQMCCl3_oL-_e5FEpul-6QKIUE6Ul5FqsOGnXA/edit?usp=sharing" rel="nofollow"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> of this TotT episode and post it in your office.</span></span><br /> <span style="font-family: "times" , "times new roman" , serif; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"><span style="font-style: normal;">By Ben Yu</span></span><br /> <br /> <span style="font-family: "times" , "times new roman" , serif;">The following test mocks out a service call to </span><span style="font-family: "courier new" , "courier" , monospace;">CloudService</span><span style="font-family: "times" , "times new roman" , serif;"><span style="font-family: "times" , "times new roman" , serif;">. </span><span style="color: purple; font-family: "times" , "times new roman" , serif;"> </span></span><span style="font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: "times" , "times new roman" , serif;">Does the test provide enough confidence that the service call is likely to work?</span></span><br /> <br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #cee2f3; vertical-align: top; width: 607px;"><pre style="background-color: #cee2f3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void uploadFileToCloudStorage() { when(<span style="font-weight: bold;">mockCloudService</span>.write( WriteRequest.newBuilder().setUserId(“<span style="font-weight: bold;">testuser</span>”).setFileType(“<span style="font-weight: bold;">plain/text</span>”)...)) .thenReturn(WriteResponse.newBuilder().setUploadId(“uploadId”).build()); CloudUploader <span style="font-weight: bold;">cloudUploader</span> = new CloudUploader(<span style="font-weight: bold;">mockCloudService</span>); Uri <span style="font-weight: bold;">uri</span> = <span style="font-weight: bold;">cloudUploader</span>.uploadFile(new File(“/path/to/foo.txt”)); // The uploaded file URI contains the user ID, file type, and upload ID. (Or does it?) assertThat(<span style="font-weight: bold;">uri</span>).isEqualTo(new Uri(“<span style="font-weight: bold;">/testuser/text/uploadId</span>.txt”));</span></pre> </td></tr> </tbody></table> </div> <br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">Lots of things can go wrong,</span></span><span style="font-family: times, times new roman, serif;"> especially when service contracts get complex. For example, </span></span><span style="font-family: "courier new" , "courier" , monospace;">plain/text</span><span style="font-family: "times" , "times new roman" , serif;"> may not be a valid file type, and you can’t verify that the URI of the uploaded file is correct.</span><br /> <br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">If the code under test relies on the contract of a service, prefer exercising the service call</span></span><span style="font-family: times, times new roman, serif;"> instead of mocking it out. This gives you more confidence that you are using the service correctly:</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void uploadFileToCloudStorage() { CloudUploader <span style="font-weight: bold;">cloudUploader</span> = new CloudUploader(<span style="font-weight: bold;">cloudService</span>); Uri <span style="font-weight: bold;">uri</span> = <span style="font-weight: bold;">cloudUploader</span>.uploadFile(”/path/to/foo.txt”); assertThat(<span style="font-weight: bold;">cloudService</span>.retrieveFile(<span style="font-weight: bold;">uri</span>)).isEqualTo(readContent(“/path/to/foo.txt")); }</span></pre> </td></tr> </tbody></table> </div> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span style="font-family: "times" , "times new roman" , serif;">How can you exercise the service call?</span><br /> <br /> <ol> <li><span style="font-family: "times" , "times new roman" , serif;"><span id="docs-internal-guid-d7ce8871-7fff-90fd-33fd-d97353a64fb8"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">Use a fake</span></span></span><span style="font-family: times, times new roman, serif;">.  A fake is a fast and lightweight implementation of the service that behaves just like the real implementation. A fake is usually maintained by the service owners; don’t create your own fake unless you can ensure its behavior will stay in sync with the real implementation.  Learn more about fakes at </span><a href="http://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html" style="font-family: times, "times new roman", serif;">testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html</a><span style="font-family: times, times new roman, serif;">.</span></span></li> <li><span style="font-family: "times" , "times new roman" , serif;"><span id="docs-internal-guid-ad45d921-7fff-e561-02d5-1a8cf098db47"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">Use a hermetic server</span></span></span><span style="font-family: times, times new roman, serif;">.  This is a real server that is brought up by the test and runs on the same machine that the test is running on. A downside of using a hermetic server is that starting it up and interacting with it can slow down tests.  Learn more about hermetic servers at </span><a href="http://testing.googleblog.com/2012/10/hermetic-servers.html" style="font-family: times, "times new roman", serif;">testing.googleblog.com/2012/10/hermetic-servers.html</a><span style="font-family: times, times new roman, serif;">.</span></span></li> </ol> <div> <span style="font-family: "times" , "times new roman" , serif;"><span style="font-family: times, times new roman, serif;">If the service you are using doesn’t have a fake or hermetic server, mocks may be the only tool at your disposal. But </span><span id="docs-internal-guid-648cf637-7fff-283e-ed04-f2bd572a8e54"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple; font-family: Times, Times New Roman, serif;">if your tests are not exercising the service call contract, you must take extra care to ensure the service call works</span></span></span><span style="font-family: times, times new roman, serif;">, such as by having a comprehensive suite of end-to-end tests or resorting to manual QA (which can be inefficient and hard to scale).</span></span></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Exercise Service Call Contracts in Tests&url=https://testing.googleblog.com/2018/11/testing-on-toilet-exercise-service-call.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/11/testing-on-toilet-exercise-service-call.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/11/testing-on-toilet-exercise-service-call.html#comments' style='font-weight: 500; text-decoration: underline;'>3 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/11/testing-on-toilet-exercise-service-call.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Ben%20Yu' rel='tag'> Ben Yu </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='4326917295936432580' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/09/efficacy-presubmit.html' itemprop='url' title='Efficacy Presubmit'> Efficacy Presubmit </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Monday, September 17, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <span style="font-family: inherit;">By Peter Spragins</span><br /> <span style="font-family: inherit;">with input from John Roane, Collin Johnston, Rose Rodrigues and Dave Chen</span><br /> <span style="font-family: inherit;"><br /></span> <span style="font-family: inherit;"> </span> <br /> <h2> <span style="font-family: inherit;"><span style="font-family: inherit; font-size: x-large;">A Brief History of Efficacy</span></span></h2> <br /> <span style="font-family: inherit;"> </span> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span></span> <span style="font-family: inherit;"><span style="font-family: inherit;">Originally named "Test Efficacy", a small team was formed in 2014 to quantify the value of individual tests to the development process. Some tests were particularly valuable because they provided a reliable breakage signal for critical code. Some tests were not useful because they were non-deterministic or they never failed. Confoundingly, tests would change in value over time as well. The team’s initial intention was to present this information to developers and help them optimize the development process.</span></span><br /> <br /> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span> </span><br /> <span style="font-family: inherit;"><span style="font-family: inherit;">To achieve the goal of informing developers about their tests, the team had to collect a huge amount of developer infrastructure/workflow data from a variety of sources across Google. Collecting all of this data in one place turned out to be incredibly valuable.</span></span><br /> <br /> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span> </span><br /> <span style="font-family: inherit;"><span style="font-family: inherit;">In addition to collecting and processing the data, the team developed a somewhat radical philosophy towards running tests at scale: the only important results come from tests which deterministically fail. Running an additional test that you know will pass is not a valuable signal to developers, and likely a waste of resources.</span></span><br /> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span></span> <br /> <div> <h2> <span style="font-family: inherit; font-size: x-large;">Background on Google Presubmit</span></h2> </div> <div> <span style="font-family: inherit;"><br /></span> <span style="font-family: inherit;">The process of committing code at Google has several testing stages. Perhaps the three most important testing stages are:</span></div> <div> <ol> <li><span style="font-family: inherit;">Individual ad-hoc testing</span></li> <li><span style="font-family: inherit;">Presubmit</span></li> <li><span style="font-family: inherit;">Continuous build/continuous integration (hereafter referred to as continuous build).</span></li> </ol> <div> <div> <span style="font-family: inherit;">Stages 1 and 2 can actually be interleaved in any order and repeated any number of times.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">A presubmit executes all of the tests which are known to be affected by the edited code within one user's proposed code changes. The "affected tests" are calculated with the help of a "project definition", a configuration maintained by teams. A presubmit can run at any point during the change proposal process, but most importantly it must run before a user can permanently commit their changes.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">Continuous build, (3), is the continuous running of all tests within a project at the newest committed version of the code. Continuous build will execute tests even when they have already passed at presubmit.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">The same test may run several times at presubmit during the development process, one last time at presubmit before a commit and then finally once again at continuous build, after being merged into the main branch of Google's huge repository. For this reason, a "missed failure" at presubmit is not a critical failure. The test will still be run at continuous build, and then rolled back if it fails.</span><br /> <span style="font-family: inherit;"><br /></span></div> </div> </div> <div> <h2> <span style="font-family: inherit; font-size: x-large;">Efficacy Presubmit Service</span></h2> </div> <div> <div> <span style="font-family: inherit;"><br /></span> <span style="font-family: inherit;">Efficacy Presubmit Service is the fusion of "running the right tests at the right time" with one of the largest collections of test/developer data in the world. The service has one simple job: save time and resources by not running, or even compiling, tests that we are very confident will pass at Presubmit. The ideal "Efficacy Presubmit" would predict which tests will pass ahead of time and only run tests which were going to fail. Then the user can get feedback from the failing tests, and fix their mistakes with the minimal possible cost of user and CPU time.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">To make this idea possible we have made one significant abstraction of the actual presubmit testing process. In a given presubmit there may be zero tests run, or many. In a presubmit with one test, if that test fails then the presubmit fails. In a presubmit with a thousand tests, only one failing test will still fail the presubmit. Efficacy Presubmit makes the abstraction that each of these test executions is an equivalent unit. This greatly simplifies creating a training dataset.</span></div> </div> <div> <br /> <br /></div> <div> <h3> <span style="color: #0b5394; font-family: inherit; font-size: large;">Machine Learning / Probabilistic Safety</span></h3> </div> <div> <span style="color: #0b5394; font-family: inherit; font-size: large;"><br /></span> <br /> <h4> <span style="color: black; font-family: inherit; font-size: medium;">Quick background on ML</span></h4> </div> <div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">ML techniques and processes are quite well known throughout the industry at this point. The Tensorflow tutorials are </span><a href="https://www.tensorflow.org/tutorials/keras/" style="text-decoration-line: none;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">a great introduction</span></a><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">. The type of ML we use is classification. A classifier is essentially a mapping from the domain of the dataset, to the range of the classes.</span><span style="color: #434343; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"> </span><a href="https://en.wikipedia.org/wiki/MNIST_database" style="text-decoration-line: none;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Mnist</span></a><span style="color: #434343; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="vertical-align: baseline; white-space: pre-wrap;">is a very famous example of classification. An mnist classifier maps from the domain of the input image to the range of digits {0, 1, …, 9}.</span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="color: #434343; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"></span></span><br /></span></span> <br /> <div dir="ltr" style="font-size: 12pt; line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt; white-space: pre-wrap;"> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">In some other classification problems, the inputs are more "</span><a href="https://en.wikipedia.org/wiki/Table_(information)" style="color: #434343;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline;">tabular</span></a><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">". A famous example of tabular classification is </span><a href="https://www.kaggle.com/uciml/iris" style="color: #434343;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline;">Iris Species</span></a><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">. This is very similar to what Efficacy does.</span></span></span></span></span><br /> <span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><br /></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"> </span></span></span></span> <br /> <div> <h4> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="color: black; font-size: medium;">Efficacy's Application of ML</span></span></span></span></span></span></h4> </div> <div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt; white-space: pre-wrap;"> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"><span style="font-family: inherit; vertical-align: baseline;">Given the abstraction on the presubmit testing process described above, predicting the outcomes of automated testing at a large company is a perfect machine learning problem in many ways. You have:</span></span></span></span></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"> </span></span> </span></span></span></span><br /> <div> <ol><span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"><span style="font-family: inherit;"> </span> <li><span style="color: #434343;"><span style="font-family: inherit; white-space: pre-wrap;">The set of test executions and results is a very large labelled dataset</span></span></li> <li><span style="font-family: inherit;">Copious numerical feature columns with trustworthy values</span></li> <ol style="list-style-type: lower-alpha;"> <li><span style="font-family: inherit;"> Recent failure history of each test</span></li> <li><span style="font-family: inherit;"> Various "distance" metrics from edited source files to tests - i.e. is this a test for the edited code?</span></li> <li><span style="font-family: inherit;">Test size and runtime data</span></li> </ol> <li><span style="font-family: inherit;">Several dimensions that can be aggregated</span></li> </span></span></span></span></span></span></ol> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c" style="font-family: inherit;">There are some aspects of the problem which make ML difficult as well:</span></span></span></span></span></span></div> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"><br /></span></span></span></span></span></span></div> </div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"> </span></span></span></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"> </span></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"> </span></span></div> <div> <ol> <li><span style="font-family: inherit;">The classes are highly imbalanced with respect to labels (the vast majority of tests are going to pass, not fail)</span></li> <li><span id="docs-internal-guid-5189d992-7fff-1821-4e3e-94f9c16f1174"><span style="font-family: inherit;"><a href="https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html" style="text-decoration-line: none;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Flaky tests</span></a><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"> can mislead the model because their labels are "untrue"</span></span></span></li> </ol> <div> <span style="color: #434343; font-family: inherit;"></span><br /> <div> <span style="color: #434343; font-family: inherit;"><span style="font-family: inherit; white-space: pre-wrap;">We chose to reduce the problem to binary classification. The model chooses whether or not to run the test. In other words, failure is the positive class, and everything else is the negative class.</span></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div> <span style="color: #434343; font-family: inherit;"><span style="white-space: pre-wrap;"><br /></span></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div> <span style="color: #434343; font-family: inherit;"><span style="white-space: pre-wrap;">We pick a threshold that results in an extremely low number of false negatives - failing tests which are not run because the model thinks they would have passed. This does reduce the number of skipped tests, true negatives, in exchange for a very high margin of safety. In addition to this, tests will be run afterwards at continuous build anyway, making presubmit skipping very safe.</span></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div style="color: #434343; white-space: pre-wrap;"> <span style="color: #434343; font-family: inherit;"><br /></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div> <h3> <span style="color: #434343; font-family: inherit;"> <span style="white-space: pre-wrap;"><span style="color: #0b5394; font-size: large;">Difficulties of Scale</span></span></span></h3> </div> <span style="color: #434343; font-family: inherit;"> </span></div> </div> <div> <span style="font-family: inherit;">In addition to the problems that were natural to the "schema" of the dataset, we faced some problems due to the scale of Google's testing.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"></span></span><br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Many of these problems stem from the fact that Google works out of one large repository </span><span style="background-color: white; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">(</span><a href="https://cacm.acm.org/magazines/2016/7/204032-why-google-stores-billions-of-lines-of-code-in-a-single-repository/fulltext" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">paper</span></a><span style="background-color: white; color: #222222; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">, </span><a href="https://www.youtube.com/watch?v=W71BTkUbdqE" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">talk</span></a><span style="background-color: white; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">)</span><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">. Because of this some presubmits have a very large number of tests and some commits require a large number of presubmits before they are finished. This means that the service has to make predictions for a very large number of tests all at once. If a presubmit tried to run every test at Google, then the service would have to predict each test individually. That means N times the number of columns, etc. Loading the data to generate all of these feature values uses a lot of memory.</span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Another difficulty of doing this work at scale is that even with very rare false negatives, they will still happen somewhat frequently. This requires our team to be open to communication with any customer team. In some cases we may have to tell them they were the victim of a very low probability event. In other cases we may find a bug, or room for improvement.</span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <br /></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <h3> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="color: #0b5394; font-size: large;">Results</span></span></span></span></h3> </div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span style="color: #434343;"><span style="white-space: pre-wrap;">The two key numbers for the system's performance are sensitivity, the percentage of failing tests we actually execute, and specificity, the percentage of passing tests we actually skip. The two numbers go hand in hand. For a given model, requiring a higher sensitivity will result in a lower specificity, or vice versa. We can easily tune the percentage of tests skipped, resulting in changes to the fidelity of the testing signal the developers receive. When the system is wrong, it can have some negative impact to developers if the prediction is a false negative. Rarely, it will allow a developer to commit code that will break a test during continuous build. This results in a broken "project", which takes some time to detect, and then a roll-back of the code. This requires some developer time, and a flexible mentality towards testing. In order to achieve a positive balance from this, we must extract millions of skipped tests for every negative developer experience. The sensitivity of our system is very high, and our specificity is around 25%.</span> </span></span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="color: #434343;"><br /></span></span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span></div> <div> <br /></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <span style="font-family: inherit;">By Peter Spragins</span><br /> <span style="font-family: inherit;">with input from John Roane, Collin Johnston, Rose Rodrigues and Dave Chen</span><br /> <span style="font-family: inherit;"><br /></span> <span style="font-family: inherit;"> </span> <br /> <h2> <span style="font-family: inherit;"><span style="font-family: inherit; font-size: x-large;">A Brief History of Efficacy</span></span></h2> <br /> <span style="font-family: inherit;"> </span> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span></span> <span style="font-family: inherit;"><span style="font-family: inherit;">Originally named "Test Efficacy", a small team was formed in 2014 to quantify the value of individual tests to the development process. Some tests were particularly valuable because they provided a reliable breakage signal for critical code. Some tests were not useful because they were non-deterministic or they never failed. Confoundingly, tests would change in value over time as well. The team’s initial intention was to present this information to developers and help them optimize the development process.</span></span><br /> <br /> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span> </span><br /> <span style="font-family: inherit;"><span style="font-family: inherit;">To achieve the goal of informing developers about their tests, the team had to collect a huge amount of developer infrastructure/workflow data from a variety of sources across Google. Collecting all of this data in one place turned out to be incredibly valuable.</span></span><br /> <br /> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span> </span><br /> <span style="font-family: inherit;"><span style="font-family: inherit;">In addition to collecting and processing the data, the team developed a somewhat radical philosophy towards running tests at scale: the only important results come from tests which deterministically fail. Running an additional test that you know will pass is not a valuable signal to developers, and likely a waste of resources.</span></span><br /> <span style="font-family: inherit;"><span style="font-family: inherit;"><br /></span></span> <br /> <div> <h2> <span style="font-family: inherit; font-size: x-large;">Background on Google Presubmit</span></h2> </div> <div> <span style="font-family: inherit;"><br /></span> <span style="font-family: inherit;">The process of committing code at Google has several testing stages. Perhaps the three most important testing stages are:</span></div> <div> <ol> <li><span style="font-family: inherit;">Individual ad-hoc testing</span></li> <li><span style="font-family: inherit;">Presubmit</span></li> <li><span style="font-family: inherit;">Continuous build/continuous integration (hereafter referred to as continuous build).</span></li> </ol> <div> <div> <span style="font-family: inherit;">Stages 1 and 2 can actually be interleaved in any order and repeated any number of times.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">A presubmit executes all of the tests which are known to be affected by the edited code within one user's proposed code changes. The "affected tests" are calculated with the help of a "project definition", a configuration maintained by teams. A presubmit can run at any point during the change proposal process, but most importantly it must run before a user can permanently commit their changes.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">Continuous build, (3), is the continuous running of all tests within a project at the newest committed version of the code. Continuous build will execute tests even when they have already passed at presubmit.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">The same test may run several times at presubmit during the development process, one last time at presubmit before a commit and then finally once again at continuous build, after being merged into the main branch of Google's huge repository. For this reason, a "missed failure" at presubmit is not a critical failure. The test will still be run at continuous build, and then rolled back if it fails.</span><br /> <span style="font-family: inherit;"><br /></span></div> </div> </div> <div> <h2> <span style="font-family: inherit; font-size: x-large;">Efficacy Presubmit Service</span></h2> </div> <div> <div> <span style="font-family: inherit;"><br /></span> <span style="font-family: inherit;">Efficacy Presubmit Service is the fusion of "running the right tests at the right time" with one of the largest collections of test/developer data in the world. The service has one simple job: save time and resources by not running, or even compiling, tests that we are very confident will pass at Presubmit. The ideal "Efficacy Presubmit" would predict which tests will pass ahead of time and only run tests which were going to fail. Then the user can get feedback from the failing tests, and fix their mistakes with the minimal possible cost of user and CPU time.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span style="font-family: inherit;">To make this idea possible we have made one significant abstraction of the actual presubmit testing process. In a given presubmit there may be zero tests run, or many. In a presubmit with one test, if that test fails then the presubmit fails. In a presubmit with a thousand tests, only one failing test will still fail the presubmit. Efficacy Presubmit makes the abstraction that each of these test executions is an equivalent unit. This greatly simplifies creating a training dataset.</span></div> </div> <div> <br /> <br /></div> <div> <h3> <span style="color: #0b5394; font-family: inherit; font-size: large;">Machine Learning / Probabilistic Safety</span></h3> </div> <div> <span style="color: #0b5394; font-family: inherit; font-size: large;"><br /></span> <br /> <h4> <span style="color: black; font-family: inherit; font-size: medium;">Quick background on ML</span></h4> </div> <div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">ML techniques and processes are quite well known throughout the industry at this point. The Tensorflow tutorials are </span><a href="https://www.tensorflow.org/tutorials/keras/" style="text-decoration-line: none;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">a great introduction</span></a><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">. The type of ML we use is classification. A classifier is essentially a mapping from the domain of the dataset, to the range of the classes.</span><span style="color: #434343; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"> </span><a href="https://en.wikipedia.org/wiki/MNIST_database" style="text-decoration-line: none;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Mnist</span></a><span style="color: #434343; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="vertical-align: baseline; white-space: pre-wrap;">is a very famous example of classification. An mnist classifier maps from the domain of the input image to the range of digits {0, 1, …, 9}.</span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="color: #434343; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"></span></span><br /></span></span> <br /> <div dir="ltr" style="font-size: 12pt; line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt; white-space: pre-wrap;"> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">In some other classification problems, the inputs are more "</span><a href="https://en.wikipedia.org/wiki/Table_(information)" style="color: #434343;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline;">tabular</span></a><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">". A famous example of tabular classification is </span><a href="https://www.kaggle.com/uciml/iris" style="color: #434343;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline;">Iris Species</span></a><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">. This is very similar to what Efficacy does.</span></span></span></span></span><br /> <span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span style="font-size: 12pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><br /></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"> </span></span></span></span> <br /> <div> <h4> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="color: black; font-size: medium;">Efficacy's Application of ML</span></span></span></span></span></span></h4> </div> <div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt; white-space: pre-wrap;"> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"><span style="font-family: inherit; vertical-align: baseline;">Given the abstraction on the presubmit testing process described above, predicting the outcomes of automated testing at a large company is a perfect machine learning problem in many ways. You have:</span></span></span></span></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"> </span></span> </span></span></span></span><br /> <div> <ol><span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"><span style="font-family: inherit;"> </span> <li><span style="color: #434343;"><span style="font-family: inherit; white-space: pre-wrap;">The set of test executions and results is a very large labelled dataset</span></span></li> <li><span style="font-family: inherit;">Copious numerical feature columns with trustworthy values</span></li> <ol style="list-style-type: lower-alpha;"> <li><span style="font-family: inherit;"> Recent failure history of each test</span></li> <li><span style="font-family: inherit;"> Various "distance" metrics from edited source files to tests - i.e. is this a test for the edited code?</span></li> <li><span style="font-family: inherit;">Test size and runtime data</span></li> </ol> <li><span style="font-family: inherit;">Several dimensions that can be aggregated</span></li> </span></span></span></span></span></span></ol> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c" style="font-family: inherit;">There are some aspects of the problem which make ML difficult as well:</span></span></span></span></span></span></div> <div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"><br /></span></span></span></span></span></span></div> </div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-d1dd62f4-7fff-9b0b-0e5a-8cc1a29d0a8c"> </span></span></span></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span id="docs-internal-guid-6f9a3c4d-7fff-4fa0-52a9-ee4a5ac6286c"> </span></span></span></span></div> <span id="docs-internal-guid-b7ec0cb1-7fff-cab0-2ce3-f3945d9bdc6c"><span style="font-family: inherit;"> </span></span></div> <div> <ol> <li><span style="font-family: inherit;">The classes are highly imbalanced with respect to labels (the vast majority of tests are going to pass, not fail)</span></li> <li><span id="docs-internal-guid-5189d992-7fff-1821-4e3e-94f9c16f1174"><span style="font-family: inherit;"><a href="https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html" style="text-decoration-line: none;"><span style="color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Flaky tests</span></a><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"> can mislead the model because their labels are "untrue"</span></span></span></li> </ol> <div> <span style="color: #434343; font-family: inherit;"></span><br /> <div> <span style="color: #434343; font-family: inherit;"><span style="font-family: inherit; white-space: pre-wrap;">We chose to reduce the problem to binary classification. The model chooses whether or not to run the test. In other words, failure is the positive class, and everything else is the negative class.</span></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div> <span style="color: #434343; font-family: inherit;"><span style="white-space: pre-wrap;"><br /></span></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div> <span style="color: #434343; font-family: inherit;"><span style="white-space: pre-wrap;">We pick a threshold that results in an extremely low number of false negatives - failing tests which are not run because the model thinks they would have passed. This does reduce the number of skipped tests, true negatives, in exchange for a very high margin of safety. In addition to this, tests will be run afterwards at continuous build anyway, making presubmit skipping very safe.</span></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div style="color: #434343; white-space: pre-wrap;"> <span style="color: #434343; font-family: inherit;"><br /></span></div> <span style="color: #434343; font-family: inherit;"> </span> <br /> <div> <h3> <span style="color: #434343; font-family: inherit;"> <span style="white-space: pre-wrap;"><span style="color: #0b5394; font-size: large;">Difficulties of Scale</span></span></span></h3> </div> <span style="color: #434343; font-family: inherit;"> </span></div> </div> <div> <span style="font-family: inherit;">In addition to the problems that were natural to the "schema" of the dataset, we faced some problems due to the scale of Google's testing.</span></div> <div> <span style="font-family: inherit;"><br /></span></div> <div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"></span></span><br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Many of these problems stem from the fact that Google works out of one large repository </span><span style="background-color: white; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">(</span><a href="https://cacm.acm.org/magazines/2016/7/204032-why-google-stores-billions-of-lines-of-code-in-a-single-repository/fulltext" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">paper</span></a><span style="background-color: white; color: #222222; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">, </span><a href="https://www.youtube.com/watch?v=W71BTkUbdqE" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">talk</span></a><span style="background-color: white; font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">)</span><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">. Because of this some presubmits have a very large number of tests and some commits require a large number of presubmits before they are finished. This means that the service has to make predictions for a very large number of tests all at once. If a presubmit tried to run every test at Google, then the service would have to predict each test individually. That means N times the number of columns, etc. Loading the data to generate all of these feature values uses a lot of memory.</span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-size: 12pt; vertical-align: baseline; white-space: pre-wrap;">Another difficulty of doing this work at scale is that even with very rare false negatives, they will still happen somewhat frequently. This requires our team to be open to communication with any customer team. In some cases we may have to tell them they were the victim of a very low probability event. In other cases we may find a bug, or room for improvement.</span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <br /></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <h3> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="color: #0b5394; font-size: large;">Results</span></span></span></span></h3> </div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span style="color: #434343;"><span style="white-space: pre-wrap;">The two key numbers for the system's performance are sensitivity, the percentage of failing tests we actually execute, and specificity, the percentage of passing tests we actually skip. The two numbers go hand in hand. For a given model, requiring a higher sensitivity will result in a lower specificity, or vice versa. We can easily tune the percentage of tests skipped, resulting in changes to the fidelity of the testing signal the developers receive. When the system is wrong, it can have some negative impact to developers if the prediction is a false negative. Rarely, it will allow a developer to commit code that will break a test during continuous build. This results in a broken "project", which takes some time to detect, and then a roll-back of the code. This requires some developer time, and a flexible mentality towards testing. In order to achieve a positive balance from this, we must extract millions of skipped tests for every negative developer experience. The sensitivity of our system is very high, and our specificity is around 25%.</span> </span></span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span> <br /> <div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="color: #434343;"><br /></span></span></span></span></div> <span id="docs-internal-guid-9245b746-7fff-1d3e-9a88-b7ab4fa91c9d"><span style="font-family: inherit;"> </span></span></div> <div> <br /></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Efficacy Presubmit&url=https://testing.googleblog.com/2018/09/efficacy-presubmit.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/09/efficacy-presubmit.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/09/efficacy-presubmit.html#comments' style='font-weight: 500; text-decoration: underline;'>13 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/09/efficacy-presubmit.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Peter%20Spragins' rel='tag'> Peter Spragins </a> , <a class='label' href='https://testing.googleblog.com/search/label/Project%20Showcase' rel='tag'> Project Showcase </a> </span> </div> </div> </div> <div class='post' data-id='7503626774796342483' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/07/code-health-make-interfaces-hard-to.html' itemprop='url' title='Code Health: Make Interfaces Hard to Misuse'> Code Health: Make Interfaces Hard to Misuse </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Wednesday, July 25, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <span id="docs-internal-guid-31239969-d272-57b1-3d6f-993ac1ed4155" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">This is another post in our </span><a href="https://testing.googleblog.com/2017/04/code-health-googles-internal-code.html" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Code Health</span></a><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> series. A version of this post originally appeared in Google bathrooms worldwide as a Google </span><a href="https://testing.googleblog.com/2007/01/introducing-testing-on-toilet.html" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Testing on the Toilet</span></a><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> episode. You can download a </span><a href="https://docs.google.com/document/d/1GaMJ4e2Z7XibnYtSkA3_WvVQ_2zoR7S_oHkW2jJpqDE/edit?usp=sharing" rel="nofollow" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> to display in your office. </span></span><br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; vertical-align: baseline; white-space: pre-wrap;">By Marek Kiszkis</span></span><br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span> <br /> <div dir="ltr" style="line-height: 1.2; margin-bottom: 8pt; margin-top: 0pt;"> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">We all try to avoid errors in our code. But what about errors created by callers of your code? A good interface design can make it easy for callers to do the right thing, and hard for callers to do the wrong thing. </span><span style="background-color: transparent; font-style: normal; font-variant: normal; font-weight: 700; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">Don't push the responsibility of maintaining invariants required by your class on to its callers.</span></span></span></div> <span id="docs-internal-guid-4df0c685-d286-1fe9-e163-7ef142e954fb"></span><span style="font-family: "times" , "times new roman" , serif;"> <span id="docs-internal-guid-e9a2f78f-d273-9b09-5695-86e07c3d6224"><span style="vertical-align: baseline; white-space: pre-wrap;">Can you see the issues that can arise with this code?</span></span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">class Vector { explicit <span style="font-weight: bold;">Vector</span>(int num_slots); // Creates an empty vector with `num_slots` slots. int <span style="font-weight: bold;">RemainingSlots</span>() const; // Returns the number of currently remaining slots. void <span style="font-weight: bold;">AddSlots</span>(int num_slots); // Adds `num_slots` more slots to the vector. // Adds a new element at the end of the vector. Caller must ensure that RemainingSlots() // returns at least 1 before calling this, otherwise caller should call AddSlots(). void <span style="font-weight: bold;">Insert</span>(int value); } </span></pre> </td></tr> </tbody></table> </div> <span style="font-family: "times new roman"; font-size: 13pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">If the caller forgets to call </span><span style="color: #00931e; font-family: "courier new" , "courier" , monospace; vertical-align: baseline; white-space: pre-wrap;">AddSlots()</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">, </span><span style="color: purple; font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">undefined behavior might be triggered</span><span style="color: #980000; font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">when </span><span style="color: #00931e; font-family: "courier new" , "courier" , monospace; vertical-align: baseline; white-space: pre-wrap;">Insert()</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;"> is called. The interface pushes complexity onto the caller, exposing the caller to implementation details.</span><br /> <span style="font-family: "times new roman"; font-size: 13pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">Since maintaining the slots is not relevant to the caller-visible behaviors of the class, don't expose them in the interface; </span><span style="color: purple; font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">make it impossible to trigger undefined behavior</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;"> by adding slots as needed in </span><span style="color: #00931e; font-family: "courier new" , "courier" , monospace; vertical-align: baseline; white-space: pre-wrap;">Insert()</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">.</span><span id="docs-internal-guid-dbfef7ae-d27a-fe35-f91e-3e483c0c5757"></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">class Vector { explicit <span style="font-weight: bold;">Vector</span>(int num_slots); // Adds a new element at the end of the vector. If necessary, // allocates new slots to ensure that there is enough storage // for the new value. void <span style="font-weight: bold;">Insert</span>(int value); }</span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-1e587b0e-d27d-6aed-6be0-acff454eb88f"></span><br /> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 8pt;"> <span style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Contracts enforced by the compiler are usually better than contracts enforced by runtime checks, or worse, documentation-only contracts that rely on callers to do the right thing. </span></span></div> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 8pt;"> <span style="font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: "times" , "times new roman" , serif;">Here are other examples that could signal that an interface is easy to misuse:</span></span></div> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 8pt;"> </div> <ul> <li><span style="white-space: pre-wrap;"><b><span id="docs-internal-guid-1d1a60d3-d281-2571-7a40-d9f1438328fc" style="font-weight: normal;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span style="font-family: "times" , "times new roman" , serif;">Requiring callers to call an initialization function (alternative: expose factory methods that return your object fully initialized).</span></span></span></b></span></li> <li><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Requiring callers to perform custom cleanup (alternative: use language-specific constructs that ensure automated cleanup when your object goes out of scope).</span></span></li> <li><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Allowing code paths that create objects without required parameters (e.g. a user without an ID).</span></span></li> <li><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-9bcfb1b7-d281-e948-3630-7241750bd866"><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline;">Allowing parameters for which only some values are valid, especially if it is possible to use a more appropriate type (e.g. prefer </span><span style="color: #00931e; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">Duration timeout</span></span><span style="color: #00931e; font-family: "times" , "times new roman" , serif; vertical-align: baseline;"> </span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline;">instead of </span><span style="color: #00931e; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">int timeout_in_millis</span></span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline;">). </span></span></span></li> </ul> <div> <span id="docs-internal-guid-513985a7-d283-475f-4fce-615f7d3a6957"><span style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">It is not always practical to have a foolproof interface. </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">In certain cases, relying on static analysis or documentation is necessary</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> since some requirements are impossible to express in an interface (e.g. that a callback function needs to be thread-safe). </span></span></span></div> <div> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span></div> <div> <span id="docs-internal-guid-e6a18413-d283-6836-7054-8b987cc12163"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Don’t enforce what you don’t need to enforce - avoid code that is too defensive. For example, extensive validation of function parameters can increase complexity and reduce performance.</span></span></span></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <span id="docs-internal-guid-31239969-d272-57b1-3d6f-993ac1ed4155" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">This is another post in our </span><a href="https://testing.googleblog.com/2017/04/code-health-googles-internal-code.html" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Code Health</span></a><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> series. A version of this post originally appeared in Google bathrooms worldwide as a Google </span><a href="https://testing.googleblog.com/2007/01/introducing-testing-on-toilet.html" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Testing on the Toilet</span></a><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> episode. You can download a </span><a href="https://docs.google.com/document/d/1GaMJ4e2Z7XibnYtSkA3_WvVQ_2zoR7S_oHkW2jJpqDE/edit?usp=sharing" rel="nofollow" style="text-decoration-line: none;"><span style="background-color: white; color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> to display in your office. </span></span><br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; vertical-align: baseline; white-space: pre-wrap;">By Marek Kiszkis</span></span><br /> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span> <br /> <div dir="ltr" style="line-height: 1.2; margin-bottom: 8pt; margin-top: 0pt;"> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">We all try to avoid errors in our code. But what about errors created by callers of your code? A good interface design can make it easy for callers to do the right thing, and hard for callers to do the wrong thing. </span><span style="background-color: transparent; font-style: normal; font-variant: normal; font-weight: 700; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">Don't push the responsibility of maintaining invariants required by your class on to its callers.</span></span></span></div> <span id="docs-internal-guid-4df0c685-d286-1fe9-e163-7ef142e954fb"></span><span style="font-family: "times" , "times new roman" , serif;"> <span id="docs-internal-guid-e9a2f78f-d273-9b09-5695-86e07c3d6224"><span style="vertical-align: baseline; white-space: pre-wrap;">Can you see the issues that can arise with this code?</span></span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">class Vector { explicit <span style="font-weight: bold;">Vector</span>(int num_slots); // Creates an empty vector with `num_slots` slots. int <span style="font-weight: bold;">RemainingSlots</span>() const; // Returns the number of currently remaining slots. void <span style="font-weight: bold;">AddSlots</span>(int num_slots); // Adds `num_slots` more slots to the vector. // Adds a new element at the end of the vector. Caller must ensure that RemainingSlots() // returns at least 1 before calling this, otherwise caller should call AddSlots(). void <span style="font-weight: bold;">Insert</span>(int value); } </span></pre> </td></tr> </tbody></table> </div> <span style="font-family: "times new roman"; font-size: 13pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">If the caller forgets to call </span><span style="color: #00931e; font-family: "courier new" , "courier" , monospace; vertical-align: baseline; white-space: pre-wrap;">AddSlots()</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">, </span><span style="color: purple; font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">undefined behavior might be triggered</span><span style="color: #980000; font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">when </span><span style="color: #00931e; font-family: "courier new" , "courier" , monospace; vertical-align: baseline; white-space: pre-wrap;">Insert()</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;"> is called. The interface pushes complexity onto the caller, exposing the caller to implementation details.</span><br /> <span style="font-family: "times new roman"; font-size: 13pt; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">Since maintaining the slots is not relevant to the caller-visible behaviors of the class, don't expose them in the interface; </span><span style="color: purple; font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">make it impossible to trigger undefined behavior</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;"> by adding slots as needed in </span><span style="color: #00931e; font-family: "courier new" , "courier" , monospace; vertical-align: baseline; white-space: pre-wrap;">Insert()</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">.</span><span id="docs-internal-guid-dbfef7ae-d27a-fe35-f91e-3e483c0c5757"></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">class Vector { explicit <span style="font-weight: bold;">Vector</span>(int num_slots); // Adds a new element at the end of the vector. If necessary, // allocates new slots to ensure that there is enough storage // for the new value. void <span style="font-weight: bold;">Insert</span>(int value); }</span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-1e587b0e-d27d-6aed-6be0-acff454eb88f"></span><br /> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 8pt;"> <span style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Contracts enforced by the compiler are usually better than contracts enforced by runtime checks, or worse, documentation-only contracts that rely on callers to do the right thing. </span></span></div> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 8pt;"> <span style="font-weight: 700; white-space: pre-wrap;"><span style="color: purple; font-family: "times" , "times new roman" , serif;">Here are other examples that could signal that an interface is easy to misuse:</span></span></div> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 8pt;"> </div> <ul> <li><span style="white-space: pre-wrap;"><b><span id="docs-internal-guid-1d1a60d3-d281-2571-7a40-d9f1438328fc" style="font-weight: normal;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><span style="font-family: "times" , "times new roman" , serif;">Requiring callers to call an initialization function (alternative: expose factory methods that return your object fully initialized).</span></span></span></b></span></li> <li><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Requiring callers to perform custom cleanup (alternative: use language-specific constructs that ensure automated cleanup when your object goes out of scope).</span></span></li> <li><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Allowing code paths that create objects without required parameters (e.g. a user without an ID).</span></span></li> <li><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-9bcfb1b7-d281-e948-3630-7241750bd866"><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline;">Allowing parameters for which only some values are valid, especially if it is possible to use a more appropriate type (e.g. prefer </span><span style="color: #00931e; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">Duration timeout</span></span><span style="color: #00931e; font-family: "times" , "times new roman" , serif; vertical-align: baseline;"> </span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline;">instead of </span><span style="color: #00931e; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">int timeout_in_millis</span></span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline;">). </span></span></span></li> </ul> <div> <span id="docs-internal-guid-513985a7-d283-475f-4fce-615f7d3a6957"><span style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">It is not always practical to have a foolproof interface. </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">In certain cases, relying on static analysis or documentation is necessary</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> since some requirements are impossible to express in an interface (e.g. that a callback function needs to be thread-safe). </span></span></span></div> <div> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span></div> <div> <span id="docs-internal-guid-e6a18413-d283-6836-7054-8b987cc12163"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Don’t enforce what you don’t need to enforce - avoid code that is too defensive. For example, extensive validation of function parameters can increase complexity and reduce performance.</span></span></span></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Code Health: Make Interfaces Hard to Misuse&url=https://testing.googleblog.com/2018/07/code-health-make-interfaces-hard-to.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/07/code-health-make-interfaces-hard-to.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/07/code-health-make-interfaces-hard-to.html#comments' style='font-weight: 500; text-decoration: underline;'>4 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/07/code-health-make-interfaces-hard-to.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Code%20Health' rel='tag'> Code Health </a> , <a class='label' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis' rel='tag'> Marek Kiszkis </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='6015182196746360165' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/06/testing-on-toilet-only-verify-relevant.html' itemprop='url' title='Testing on the Toilet: Only Verify Relevant Method Arguments'> Testing on the Toilet: Only Verify Relevant Method Arguments </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, June 26, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <span id="docs-internal-guid-7e1766f2-3e0c-56eb-ab06-1655721a9901" style="font-family: "times" , "times new roman" , serif;"><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">This article was adapted from a </span><a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html" style="text-decoration-line: none;"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Google Testing on the Toilet</span></a><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> (TotT) episode. You can download a </span><a href="https://docs.google.com/document/d/1BxUCm5HuCg2_cq0iSI060iLCSMXzuudb-Ob57MhCJcw/edit" rel="nofollow" style="text-decoration-line: none;"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> of this TotT episode and post it in your office.</span></span><br /> <span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: Times, Times New Roman, serif; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">By Dillon Bly</span><br /> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">What makes this test fragile?</span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void <span style="font-weight: bold;">displayGreeting_showSpecialGreetingOnNewYearsDay</span>() { fakeClock.setTime(<span style="font-weight: bold;">NEW_YEARS_DAY</span>); fakeUser.setName("<span style="font-weight: bold;">Fake User</span>”); userGreeter.displayGreeting(); // The test will fail if userGreeter.displayGreeting() didn’t call // mockUserPrompter.updatePrompt() with these exact arguments. <span style="font-weight: bold;">verify(mockUserPrompter).updatePrompt( "Hi Fake User! Happy New Year!", TitleBar.of("2018-01-01"), PromptStyle.NORMAL);</span> }</span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-bc84817c-3e14-e5cc-a05d-16b057db019f"><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">The test specifies exact values for all arguments to </span><span style="color: #0d904f; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "courier new" , "courier" , monospace;">mockUserPrompter</span></span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">. </span><span style="font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">These arguments may need to be updated when the code under test is changed, even if the changes are unrelated to the behavior being tested</span></span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">. For example, if additional text is added to </span><span style="color: #0d904f; font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">TitleBar</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">, every test in the codebase that specifies this argument will need to be updated.</span></span><br /> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-2bb90996-3e17-07d1-2522-8f9508921c42" style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">In addition, </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">verifying too many arguments makes it difficult to understand what behavior is being tested</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> since it’s not obvious which arguments are important to the test and which are irrelevant.</span></span><br /> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-f45a3252-3e17-a749-b17d-2afb4c9bacca" style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">Instead, </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline;"><span style="color: purple;">only verify arguments that affect the correctness of the specific behavior being tested</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">. You can use argument matchers (e.g., </span><span style="color: #0d904f; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">any()</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> and </span><span style="color: #0d904f; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">contains()</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> in Mockito) to ignore arguments that don't need to be verified:</span></span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void <span style="font-weight: bold;">displayGreeting_showSpecialGreetingOnNewYearsDay</span>() { fakeClock.setTime(<span style="font-weight: bold;">NEW_YEARS_DAY</span>); userGreeter.displayGreeting(); <span style="font-weight: bold;">verify(mockUserPrompter).updatePrompt(contains("Happy New Year!"), any(), any()));</span> }</span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-f315adfc-3e1c-2c4c-701f-353cc666cc03" style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Arguments ignored in one test can be verified in other tests. Following this pattern allows us to </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">verify only one behavior per test</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">, which makes tests more readable and more resilient to change. For example, here is a separate test that we might write:</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void <span style="font-weight: bold;">displayGreeting_renderUserName</span>() { fakeUser.setName(“<span style="font-weight: bold;">Fake User</span>”); userGreeter.displayGreeting(); // Focus on the argument relevant to showing the user's name. <span style="font-weight: bold;">verify(mockUserPrompter).updatePrompt(contains("Hi Fake User!"), any(), any());</span> } </span></pre> </td></tr> </tbody></table> </div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <span id="docs-internal-guid-7e1766f2-3e0c-56eb-ab06-1655721a9901" style="font-family: "times" , "times new roman" , serif;"><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">This article was adapted from a </span><a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html" style="text-decoration-line: none;"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">Google Testing on the Toilet</span></a><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> (TotT) episode. You can download a </span><a href="https://docs.google.com/document/d/1BxUCm5HuCg2_cq0iSI060iLCSMXzuudb-Ob57MhCJcw/edit" rel="nofollow" style="text-decoration-line: none;"><span style="color: #1155cc; font-style: italic; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> of this TotT episode and post it in your office.</span></span><br /> <span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: Times, Times New Roman, serif; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">By Dillon Bly</span><br /> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">What makes this test fragile?</span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void <span style="font-weight: bold;">displayGreeting_showSpecialGreetingOnNewYearsDay</span>() { fakeClock.setTime(<span style="font-weight: bold;">NEW_YEARS_DAY</span>); fakeUser.setName("<span style="font-weight: bold;">Fake User</span>”); userGreeter.displayGreeting(); // The test will fail if userGreeter.displayGreeting() didn’t call // mockUserPrompter.updatePrompt() with these exact arguments. <span style="font-weight: bold;">verify(mockUserPrompter).updatePrompt( "Hi Fake User! Happy New Year!", TitleBar.of("2018-01-01"), PromptStyle.NORMAL);</span> }</span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-bc84817c-3e14-e5cc-a05d-16b057db019f"><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">The test specifies exact values for all arguments to </span><span style="color: #0d904f; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "courier new" , "courier" , monospace;">mockUserPrompter</span></span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">. </span><span style="font-family: "times" , "times new roman" , serif; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">These arguments may need to be updated when the code under test is changed, even if the changes are unrelated to the behavior being tested</span></span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">. For example, if additional text is added to </span><span style="color: #0d904f; font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">TitleBar</span><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">, every test in the codebase that specifies this argument will need to be updated.</span></span><br /> <span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-2bb90996-3e17-07d1-2522-8f9508921c42" style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">In addition, </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">verifying too many arguments makes it difficult to understand what behavior is being tested</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> since it’s not obvious which arguments are important to the test and which are irrelevant.</span></span><br /> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-f45a3252-3e17-a749-b17d-2afb4c9bacca" style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">Instead, </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline;"><span style="color: purple;">only verify arguments that affect the correctness of the specific behavior being tested</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">. You can use argument matchers (e.g., </span><span style="color: #0d904f; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">any()</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> and </span><span style="color: #0d904f; vertical-align: baseline;"><span style="font-family: "courier new" , "courier" , monospace;">contains()</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> in Mockito) to ignore arguments that don't need to be verified:</span></span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void <span style="font-weight: bold;">displayGreeting_showSpecialGreetingOnNewYearsDay</span>() { fakeClock.setTime(<span style="font-weight: bold;">NEW_YEARS_DAY</span>); userGreeter.displayGreeting(); <span style="font-weight: bold;">verify(mockUserPrompter).updatePrompt(contains("Happy New Year!"), any(), any()));</span> }</span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-f315adfc-3e1c-2c4c-701f-353cc666cc03" style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Arguments ignored in one test can be verified in other tests. Following this pattern allows us to </span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">verify only one behavior per test</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">, which makes tests more readable and more resilient to change. For example, here is a separate test that we might write:</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">@Test public void <span style="font-weight: bold;">displayGreeting_renderUserName</span>() { fakeUser.setName(“<span style="font-weight: bold;">Fake User</span>”); userGreeter.displayGreeting(); // Focus on the argument relevant to showing the user's name. <span style="font-weight: bold;">verify(mockUserPrompter).updatePrompt(contains("Hi Fake User!"), any(), any());</span> } </span></pre> </td></tr> </tbody></table> </div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Only Verify Relevant Method Arguments&url=https://testing.googleblog.com/2018/06/testing-on-toilet-only-verify-relevant.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/06/testing-on-toilet-only-verify-relevant.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/06/testing-on-toilet-only-verify-relevant.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/06/testing-on-toilet-only-verify-relevant.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Dillon%20Bly' rel='tag'> Dillon Bly </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='4941039424603171628' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/06/testing-on-toilet-keep-tests-focused.html' itemprop='url' title='Testing on the Toilet: Keep Tests Focused'> Testing on the Toilet: Keep Tests Focused </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Monday, June 11, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i><span style="font-family: "times" , "times new roman" , serif;">This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1QwLg7uY7vMpWiahru76o0QdCWEeV8pO69FcwX0lCKWA/edit?usp=sharing" rel="nofollow">printer-friendly version</a> of this TotT episode and post it in your office.</span></i><br /> <i><span style="font-family: "times" , "times new roman" , serif;"><br /></span></i> <span style="font-family: "times" , "times new roman" , serif;">By Ben Yu</span><br /> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span id="docs-internal-guid-7b9c23ee-ef94-09b4-cb1b-89c9a235a8c1"><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">What scenario does the following code test?</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #cfe2f3; vertical-align: top; width: 607px;"><pre style="background-color: #cfe2f3; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;">TEST_F(BankAccountTest, <span style="font-weight: bold;">WithdrawFromAccount</span>) { Transaction transaction = account_.Deposit(Usd(5)); clock_.AdvanceTime(MIN_TIME_TO_SETTLE); account_.Settle(transaction); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(5)), IsOk()); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(1)), IsRejected()); account_.SetOverdraftLimit(Usd(1)); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(1)), IsOk()); }</span></pre> </td></tr> </tbody></table> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 10pt;"> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #333333; vertical-align: baseline; white-space: pre-wrap;">Translated to English: “</span><span style="background-color: white; color: #333333; font-style: italic; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">(1)</span><span style="background-color: white; color: #333333; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> I had $5 and was able to withdraw $5; </span><span style="background-color: white; color: #333333; font-style: italic; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">(2)</span><span style="background-color: white; color: #333333; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> then got rejected when overdrawing $1;</span><span style="background-color: white; color: #333333; font-style: italic; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> (3)</span><span style="background-color: white; color: #333333; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> but if I enable overdraft with a $1 limit, I can withdraw $1.”</span><span style="background-color: white; color: #333333; vertical-align: baseline; white-space: pre-wrap;"> If that sounds a little hard to track, it is: </span><span style="background-color: white; color: purple; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">it is testing three scenarios, not one.</span></span></div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"> </span> <br /> <div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"><span style="background-color: white; color: #980000; font-size: 13pt; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span></div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"><span style="font-family: "times" , "times new roman" , serif;"> </span></span> <br /> <div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"><span style="background-color: white; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-73fc15db-ef9f-118e-1761-a33020c05953" style="font-family: "times" , "times new roman" , serif; font-weight: normal;"><span style="color: #333333; vertical-align: baseline;">A better approach is to </span><span style="color: purple; font-weight: 700; vertical-align: baseline;">exercise each scenario in its own test</span><span style="color: #333333; vertical-align: baseline;">:</span></span></span></span></div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"> </span></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">TEST_F(BankAccountTest, <span style="font-weight: bold;">CanWithdrawWithinBalance</span>) { DepositAndSettle(Usd(5)); // Common setup code is extracted into a helper method. EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(5)), IsOk()); } TEST_F(BankAccountTest, <span style="font-weight: bold;">CannotOverdraw</span>) { DepositAndSettle(Usd(5)); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(6)), IsRejected()); } TEST_F(BankAccountTest, <span style="font-weight: bold;">CanOverdrawUpToOverdraftLimit</span>) { DepositAndSettle(Usd(5)); account_.SetOverdraftLimit(Usd(1)); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(6)), IsOk()); }</span></pre> </td></tr> </tbody></table> </div> <br /> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 10pt;"> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-style: normal; font-variant: normal; font-weight: 700; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">Writing tests this way provides many benefits</span></span><span style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">:</span></span></div> <span style="font-family: "times" , "times new roman" , serif;"><span id="docs-internal-guid-7d99b1a3-efa1-48fb-89c0-831589ed64ac"></span></span><br /> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Logic is easier to understand because there is less code to read in each test method.</span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Setup code in each test is simpler because it only needs to serve a single scenario.</span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Side effects of one scenario will not accidentally invalidate or mask a later scenario’s assumptions. </span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">If a scenario in one test fails, other scenarios will still run since they are unaffected by the failure.</span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Test names clearly describe each scenario, which makes it easier to learn which scenarios exist.</span></li> </ul> <div> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 10pt;"> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">One sign that you might be testing more than one scenario: after asserting the output of one call to the system under test, the test makes another call to the system under test.</span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"> </span> <br /> <div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"></span></span><br /></span> <br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 10pt;"> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"><span style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline;"><span style="color: purple;">While a scenario for a unit test often consists of a single call to the system under test, its scope can be larger for integration and end-to-end tests</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">. For example, a test that a web UI can send email might open the inbox, click the </span><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">compose</span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> button, write some text, and press the </span><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">send</span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> button.</span></span></span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"> </span></span></span> <br /> <div style="font-size: 13pt;"> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"><span style="font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><br /></span></span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"> </span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"> </span></div> <div> <span style="font-size: 17.3333px; white-space: pre-wrap;"><br /></span></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i><span style="font-family: "times" , "times new roman" , serif;">This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1QwLg7uY7vMpWiahru76o0QdCWEeV8pO69FcwX0lCKWA/edit?usp=sharing" rel="nofollow">printer-friendly version</a> of this TotT episode and post it in your office.</span></i><br /> <i><span style="font-family: "times" , "times new roman" , serif;"><br /></span></i> <span style="font-family: "times" , "times new roman" , serif;">By Ben Yu</span><br /> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span id="docs-internal-guid-7b9c23ee-ef94-09b4-cb1b-89c9a235a8c1"><span style="font-family: "times" , "times new roman" , serif; vertical-align: baseline; white-space: pre-wrap;">What scenario does the following code test?</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #cfe2f3; vertical-align: top; width: 607px;"><pre style="background-color: #cfe2f3; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;">TEST_F(BankAccountTest, <span style="font-weight: bold;">WithdrawFromAccount</span>) { Transaction transaction = account_.Deposit(Usd(5)); clock_.AdvanceTime(MIN_TIME_TO_SETTLE); account_.Settle(transaction); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(5)), IsOk()); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(1)), IsRejected()); account_.SetOverdraftLimit(Usd(1)); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(1)), IsOk()); }</span></pre> </td></tr> </tbody></table> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 10pt;"> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #333333; vertical-align: baseline; white-space: pre-wrap;">Translated to English: “</span><span style="background-color: white; color: #333333; font-style: italic; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">(1)</span><span style="background-color: white; color: #333333; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> I had $5 and was able to withdraw $5; </span><span style="background-color: white; color: #333333; font-style: italic; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">(2)</span><span style="background-color: white; color: #333333; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> then got rejected when overdrawing $1;</span><span style="background-color: white; color: #333333; font-style: italic; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> (3)</span><span style="background-color: white; color: #333333; font-style: italic; vertical-align: baseline; white-space: pre-wrap;"> but if I enable overdraft with a $1 limit, I can withdraw $1.”</span><span style="background-color: white; color: #333333; vertical-align: baseline; white-space: pre-wrap;"> If that sounds a little hard to track, it is: </span><span style="background-color: white; color: purple; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">it is testing three scenarios, not one.</span></span></div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"> </span> <br /> <div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"><span style="background-color: white; color: #980000; font-size: 13pt; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><br /></span></span></div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"><span style="font-family: "times" , "times new roman" , serif;"> </span></span> <br /> <div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"><span style="background-color: white; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-73fc15db-ef9f-118e-1761-a33020c05953" style="font-family: "times" , "times new roman" , serif; font-weight: normal;"><span style="color: #333333; vertical-align: baseline;">A better approach is to </span><span style="color: purple; font-weight: 700; vertical-align: baseline;">exercise each scenario in its own test</span><span style="color: #333333; vertical-align: baseline;">:</span></span></span></span></div> <span id="docs-internal-guid-78e6ff07-ef9e-e0ec-fbd0-3db93bb4f28f"> </span></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">TEST_F(BankAccountTest, <span style="font-weight: bold;">CanWithdrawWithinBalance</span>) { DepositAndSettle(Usd(5)); // Common setup code is extracted into a helper method. EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(5)), IsOk()); } TEST_F(BankAccountTest, <span style="font-weight: bold;">CannotOverdraw</span>) { DepositAndSettle(Usd(5)); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(6)), IsRejected()); } TEST_F(BankAccountTest, <span style="font-weight: bold;">CanOverdrawUpToOverdraftLimit</span>) { DepositAndSettle(Usd(5)); account_.SetOverdraftLimit(Usd(1)); EXPECT_THAT(<span style="font-weight: bold;">account_.Withdraw</span>(Usd(6)), IsOk()); }</span></pre> </td></tr> </tbody></table> </div> <br /> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 10pt;"> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-style: normal; font-variant: normal; font-weight: 700; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">Writing tests this way provides many benefits</span></span><span style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">:</span></span></div> <span style="font-family: "times" , "times new roman" , serif;"><span id="docs-internal-guid-7d99b1a3-efa1-48fb-89c0-831589ed64ac"></span></span><br /> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Logic is easier to understand because there is less code to read in each test method.</span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Setup code in each test is simpler because it only needs to serve a single scenario.</span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Side effects of one scenario will not accidentally invalidate or mask a later scenario’s assumptions. </span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">If a scenario in one test fails, other scenarios will still run since they are unaffected by the failure.</span></li> </ul> <ul style="margin-bottom: 0pt; margin-top: 0pt;"> <li dir="ltr" style="background-color: transparent; color: black; font-style: normal; font-variant: normal; font-weight: 400; list-style-type: disc; margin-left: -13.5pt; text-decoration: none; vertical-align: baseline; white-space: pre;"><span style="font-family: "times" , "times new roman" , serif;">Test names clearly describe each scenario, which makes it easier to learn which scenarios exist.</span></li> </ul> <div> <div dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 10pt;"> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">One sign that you might be testing more than one scenario: after asserting the output of one call to the system under test, the test makes another call to the system under test.</span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"> </span> <br /> <div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"></span></span><br /></span> <br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 10pt;"> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"><span style="font-family: "times" , "times new roman" , serif;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline;"><span style="color: purple;">While a scenario for a unit test often consists of a single call to the system under test, its scope can be larger for integration and end-to-end tests</span></span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">. For example, a test that a web UI can send email might open the inbox, click the </span><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">compose</span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> button, write some text, and press the </span><span style="font-style: italic; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;">send</span><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"> button.</span></span></span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"> </span></span></span> <br /> <div style="font-size: 13pt;"> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"><span style="font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline;"><br /></span></span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-ca653e74-efa1-8e8c-1649-9423a50abeef"> </span></span></span></div> <span id="docs-internal-guid-b1437967-efa1-7617-055d-4f63e0f47cff"> </span></div> <div> <span style="font-size: 17.3333px; white-space: pre-wrap;"><br /></span></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Keep Tests Focused&url=https://testing.googleblog.com/2018/06/testing-on-toilet-keep-tests-focused.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/06/testing-on-toilet-keep-tests-focused.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/06/testing-on-toilet-keep-tests-focused.html#comments' style='font-weight: 500; text-decoration: underline;'>6 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/06/testing-on-toilet-keep-tests-focused.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Ben%20Yu' rel='tag'> Ben Yu </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='499881011831374904' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html' itemprop='url' title='Code Health: Understanding Code In Review'> Code Health: Understanding Code In Review </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, May 01, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <span id="docs-internal-guid-bda27e4f-1d0e-8d72-6e8d-b6375376fbc6"></span><br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">This is another post in our </span><a href="https://testing.googleblog.com/2017/04/code-health-googles-internal-code.html" style="text-decoration: none;"><span style="background-color: white; color: #1155cc; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: underline; vertical-align: baseline; white-space: pre-wrap;">Code Health</span></a><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> series. A version of this post originally appeared in Google bathrooms worldwide as a Google </span><a href="https://testing.googleblog.com/2007/01/introducing-testing-on-toilet.html" style="text-decoration: none;"><span style="background-color: white; color: #1155cc; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: underline; vertical-align: baseline; white-space: pre-wrap;">Testing on the Toilet</span></a><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> episode. You can download a </span><a href="https://docs.google.com/document/d/1fhSq_bbzlS4amLlhLbHq7VYz3VySkbfy0uKlTn2rLl4/edit?usp=sharing" rel="nofollow" style="text-decoration: none;"><span style="background-color: white; color: #1155cc; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: underline; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> to display in your office.</span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-family: "arial"; font-size: 9.5pt; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre;"><br /></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">By Max Kanat-Alexander</span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-a9eb0958-1d10-1166-426e-fc302c9fafb2"><span style="background-color: transparent; color: black; font-family: "times" , "times new roman" , serif; vertical-align: baseline;">It's easy to assume that a developer who sends you some code for review is smarter than you'll ever be, and that's why you don't understand their code.</span></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="background-color: transparent; color: black; font-size: 13pt; vertical-align: baseline;"><br /></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-a9eb0958-1d10-2b95-5767-88453750d8e8" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: transparent; color: black; vertical-align: baseline;">But in reality,</span><span style="background-color: transparent; color: #7e0021; font-weight: 700; vertical-align: baseline;"> </span><span style="background-color: transparent; color: purple; font-weight: 700; vertical-align: baseline;">if code is hard to understand, it's probably too complex</span><span style="background-color: transparent; color: black; vertical-align: baseline;">. If you're familiar with the programming language being used,</span><span style="background-color: transparent; color: #7e0021; font-weight: 700; vertical-align: baseline;"> </span><span style="background-color: transparent; color: purple; font-weight: 700; vertical-align: baseline;">reading healthy code should be almost as easy as reading a book in your native language</span><span style="background-color: transparent; color: black; vertical-align: baseline;">.</span></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="background-color: transparent; color: black; font-size: 13pt; vertical-align: baseline;"><br /></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-a9eb0958-1d10-4444-dc5c-150f5bbac905"><span style="color: black; font-family: "times" , "times new roman" , serif; vertical-align: baseline;">Pretend a developer sends you this block of Python to be reviewed:</span></span></span></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">def IsOkay(n): f = False for i in range(2, n): if n % i == 0: f = True return not f </span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-a9eb0958-1d12-e8f0-0e06-849983bd10d9" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: purple; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Don't spend more than a few seconds trying to understand it</span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;">. Simply add a code review comment saying,</span><span style="background-color: white; color: maroon; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;">"It's hard for me to understand this piece of code," or be more specific, and say,</span><span style="background-color: white; color: #7e0021; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;">"Please use more descriptive names here."</span></span><br /> <span style="background-color: white; font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-a9eb0958-1d13-04a9-5d6d-ed2044db832e" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: purple; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">The developer then clarifies the code</span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;"> and sends it to you for review again:</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #fff2cc; vertical-align: top; width: 607px;"><pre style="background-color: #fff2cc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">def IsPrime(n): for divisor in range(2, n / 2): if n % divisor == 0: return False return True </span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-a9eb0958-1d14-74ac-b4e9-3ae86e827efa"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Now we can read it pretty easily, which is a benefit in itself.</span></span></span><br /> <span style="background-color: white; font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-a9eb0958-1d14-8e8f-9ded-0a45de385efe"><span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Often, </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">just asking a developer to clarify a piece of code will result in fundamental improvements</span></span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. In this case,</span><span style="background-color: white; color: maroon; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">the developer noticed possible performance improvements since the code was easier to read—the function now returns earlier when the number isn't prime, and the loop only goes to</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span></span><span style="background-color: white; font-family: "courier new"; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">n/2</span><span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> instead of</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span></span><span style="background-color: white; font-family: "courier new"; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">n</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. </span></span><br /> <span style="background-color: white; font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-a9eb0958-1d14-b070-4ff8-2177a8f42d0c"><span style="font-family: "times" , "times new roman" , serif;"><span style="color: purple;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">However, now that we can easily understand this code,</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">we can see many problems with it</span></span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. For example, it has strange behavior with 0 and 1, and there are other problems, too. But most importantly, it is now apparent that</span><span style="background-color: white; color: #980000; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">this entire function should be removed and be replaced with a preexisting function for detecting if a number is prime. Clarifying the code helped both the developer and reviewer.</span></span></span><br /> <span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span> <span id="docs-internal-guid-a9eb0958-1d14-c904-1c55-5cf152a80551"><span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">In summary,</span><span style="background-color: white; color: #7e0021; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">don't waste time reviewing code that is hard to understand, just ask for it to be clarified</span></span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. In fact, such review comments are one of the most useful and important tools a code reviewer has!</span></span></span> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <span id="docs-internal-guid-bda27e4f-1d0e-8d72-6e8d-b6375376fbc6"></span><br /> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">This is another post in our </span><a href="https://testing.googleblog.com/2017/04/code-health-googles-internal-code.html" style="text-decoration: none;"><span style="background-color: white; color: #1155cc; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: underline; vertical-align: baseline; white-space: pre-wrap;">Code Health</span></a><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> series. A version of this post originally appeared in Google bathrooms worldwide as a Google </span><a href="https://testing.googleblog.com/2007/01/introducing-testing-on-toilet.html" style="text-decoration: none;"><span style="background-color: white; color: #1155cc; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: underline; vertical-align: baseline; white-space: pre-wrap;">Testing on the Toilet</span></a><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> episode. You can download a </span><a href="https://docs.google.com/document/d/1fhSq_bbzlS4amLlhLbHq7VYz3VySkbfy0uKlTn2rLl4/edit?usp=sharing" rel="nofollow" style="text-decoration: none;"><span style="background-color: white; color: #1155cc; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: underline; vertical-align: baseline; white-space: pre-wrap;">printer-friendly version</span></a><span style="background-color: white; color: #222222; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> to display in your office.</span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-family: "arial"; font-size: 9.5pt; font-style: italic; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre;"><br /></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">By Max Kanat-Alexander</span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-a9eb0958-1d10-1166-426e-fc302c9fafb2"><span style="background-color: transparent; color: black; font-family: "times" , "times new roman" , serif; vertical-align: baseline;">It's easy to assume that a developer who sends you some code for review is smarter than you'll ever be, and that's why you don't understand their code.</span></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="background-color: transparent; color: black; font-size: 13pt; vertical-align: baseline;"><br /></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-a9eb0958-1d10-2b95-5767-88453750d8e8" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: transparent; color: black; vertical-align: baseline;">But in reality,</span><span style="background-color: transparent; color: #7e0021; font-weight: 700; vertical-align: baseline;"> </span><span style="background-color: transparent; color: purple; font-weight: 700; vertical-align: baseline;">if code is hard to understand, it's probably too complex</span><span style="background-color: transparent; color: black; vertical-align: baseline;">. If you're familiar with the programming language being used,</span><span style="background-color: transparent; color: #7e0021; font-weight: 700; vertical-align: baseline;"> </span><span style="background-color: transparent; color: purple; font-weight: 700; vertical-align: baseline;">reading healthy code should be almost as easy as reading a book in your native language</span><span style="background-color: transparent; color: black; vertical-align: baseline;">.</span></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span style="background-color: transparent; color: black; font-size: 13pt; vertical-align: baseline;"><br /></span></span></div> <div dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"> <span style="background-color: white; color: #222222; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"><span id="docs-internal-guid-a9eb0958-1d10-4444-dc5c-150f5bbac905"><span style="color: black; font-family: "times" , "times new roman" , serif; vertical-align: baseline;">Pretend a developer sends you this block of Python to be reviewed:</span></span></span></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">def IsOkay(n): f = False for i in range(2, n): if n % i == 0: f = True return not f </span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-a9eb0958-1d12-e8f0-0e06-849983bd10d9" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: purple; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Don't spend more than a few seconds trying to understand it</span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;">. Simply add a code review comment saying,</span><span style="background-color: white; color: maroon; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;">"It's hard for me to understand this piece of code," or be more specific, and say,</span><span style="background-color: white; color: #7e0021; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;">"Please use more descriptive names here."</span></span><br /> <span style="background-color: white; font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-a9eb0958-1d13-04a9-5d6d-ed2044db832e" style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; color: purple; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">The developer then clarifies the code</span><span style="background-color: white; vertical-align: baseline; white-space: pre-wrap;"> and sends it to you for review again:</span></span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #fff2cc; vertical-align: top; width: 607px;"><pre style="background-color: #fff2cc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">def IsPrime(n): for divisor in range(2, n / 2): if n % divisor == 0: return False return True </span></pre> </td></tr> </tbody></table> </div> <br /> <span id="docs-internal-guid-a9eb0958-1d14-74ac-b4e9-3ae86e827efa"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;">Now we can read it pretty easily, which is a benefit in itself.</span></span></span><br /> <span style="background-color: white; font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-a9eb0958-1d14-8e8f-9ded-0a45de385efe"><span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Often, </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">just asking a developer to clarify a piece of code will result in fundamental improvements</span></span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. In this case,</span><span style="background-color: white; color: maroon; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">the developer noticed possible performance improvements since the code was easier to read—the function now returns earlier when the number isn't prime, and the loop only goes to</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span></span><span style="background-color: white; font-family: "courier new"; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">n/2</span><span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> instead of</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span></span><span style="background-color: white; font-family: "courier new"; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">n</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. </span></span><br /> <span style="background-color: white; font-size: 13pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span> <span id="docs-internal-guid-a9eb0958-1d14-b070-4ff8-2177a8f42d0c"><span style="font-family: "times" , "times new roman" , serif;"><span style="color: purple;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">However, now that we can easily understand this code,</span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">we can see many problems with it</span></span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. For example, it has strange behavior with 0 and 1, and there are other problems, too. But most importantly, it is now apparent that</span><span style="background-color: white; color: #980000; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">this entire function should be removed and be replaced with a preexisting function for detecting if a number is prime. Clarifying the code helped both the developer and reviewer.</span></span></span><br /> <span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: "times" , "times new roman" , serif;"><br /></span></span> <span id="docs-internal-guid-a9eb0958-1d14-c904-1c55-5cf152a80551"><span style="font-family: "times" , "times new roman" , serif;"><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">In summary,</span><span style="background-color: white; color: #7e0021; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;"><span style="color: purple;">don't waste time reviewing code that is hard to understand, just ask for it to be clarified</span></span><span style="background-color: white; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">. In fact, such review comments are one of the most useful and important tools a code reviewer has!</span></span></span> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Code Health: Understanding Code In Review&url=https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html#comments' style='font-weight: 500; text-decoration: underline;'>1 comment</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Code%20Health' rel='tag'> Code Health </a> , <a class='label' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander' rel='tag'> Max Kanat-Alexander </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='4368690219079281355' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2018/02/testing-on-toilet-cleanly-create-test.html' itemprop='url' title='Testing on the Toilet: Cleanly Create Test Data'> Testing on the Toilet: Cleanly Create Test Data </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, February 20, 2018 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i><span style="font-family: "times" , "times new roman" , serif;">This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1ntYzDmcBgs1uaRnLFqA96Ou1G9h-92pXujjENrbPa-8/edit?usp=sharing" rel="nofollow">printer-friendly version</a> of this TotT episode and post it in your office.</span></i><br /> <i><br /></i><span style="font-family: "times" , "times new roman" , serif;">By Ben Yu</span><br /> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span style="font-family: "times" , "times new roman" , serif;"><span style="color: purple; font-weight: bold;"> Helper methods make it easier to create test data. But they can become difficult to read over time</span> as you need more variations of the test data to satisfy constantly evolving requirements from new tests:</span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: #006600;">// This helper method starts with just a single parameter:</span> Company company = <span style="font-weight: bold;">newCompany</span>(PUBLIC);</span></pre> <pre style="background-color: #f4cccc; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;"> <span style="color: #006600;">// But soon it acquires more and more parameters. // Conditionals creep into the newCompany() method body to handle the nulls, // and the method calls become hard to read due to the long parameter lists: </span>Company small = <span style="font-weight: bold;">newCompany</span>(2, 2, null, PUBLIC); Company privatelyOwned = <span style="font-weight: bold;">newCompany</span>(null, null, null, PRIVATE); Company bankrupt = <span style="font-weight: bold;">newCompany</span>(null, null, PAST_DATE, PUBLIC); <span style="color: #006600;">// Or a new method is added each time a test needs a different combination of fields: </span>Company small = <span style="font-weight: bold;">newCompanyWithEmployeesAndBoardMembers</span>(2, 2, PUBLIC); Company privatelyOwned = <span style="font-weight: bold;">newCompanyWithType</span>(PRIVATE); Company bankrupt = <span style="font-weight: bold;">newCompanyWithBankruptcyDate</span>(PAST_DATE, PUBLIC);</span></pre> </td></tr> </tbody></table> </div> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span style="font-family: "times" , "times new roman" , serif;">Instead, <span style="color: purple; font-weight: bold;">use the test data builder pattern: create a helper method that returns a partially-built object</span> (e.g., a Builder in languages such as Java, or a mutable object) whose state can be overridden in tests. The helper method initializes logically-required fields to reasonable defaults, so <span style="color: purple; font-weight: bold;">each test can specify only fields relevant to the case being tested</span>: </span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">Company small = <span style="font-weight: bold;">newCompany</span>().setEmployees(2).setBoardMembers(2).build(); Company privatelyOwned = <span style="font-weight: bold;">newCompany</span>().setType(PRIVATE).build(); Company bankrupt = <span style="font-weight: bold;">newCompany</span>().setBankruptcyDate(PAST_DATE).build(); Company arbitraryCompany = <span style="font-weight: bold;">newCompany</span>().build(); </span></pre> <pre style="background-color: #d9ead3; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: blue;"> </span></span></pre> <pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: blue;">// Zero parameters makes this method reusable for different variations of Company. // It also doesn’t need conditionals to ignore parameters that aren’t set (e.g. null // values) since a test can simply not set a field if it doesn’t care about it. </span>private static <span style="font-weight: bold;">Company.Builder newCompany</span>() { return Company.newBuilder().setType(PUBLIC).setEmployees(100); <span style="color: blue;">// Set required fields</span> }</span></pre> </td></tr> </tbody></table> </div> <br /> <span style="font-family: "times" , "times new roman" , serif;">Also note that <span style="color: purple; font-weight: bold;">tests should never rely on default values that are specified by a helper method</span> since that forces readers to read the helper method’s implementation details in order to understand the test.</span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #cfe2f3; vertical-align: top; width: 607px;"><pre style="background-color: #cfe2f3; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: #006600;">// This test needs a public company, so explicitly set it. // It also needs a company with no board members, so explicitly clear it. </span>Company publicNoBoardMembers = newCompany().<span style="font-weight: bold;">setType(PUBLIC).clearBoardMembers</span>().build();</span></pre> </td></tr> </tbody></table> </div> <i><br /></i> <i><span style="font-family: "times" , "times new roman" , serif;">You can learn more about this topic at <a href="http://www.natpryce.com/articles/000714.html">http://www.natpryce.com/articles/000714.html</a></span></i> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i><span style="font-family: "times" , "times new roman" , serif;">This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1ntYzDmcBgs1uaRnLFqA96Ou1G9h-92pXujjENrbPa-8/edit?usp=sharing" rel="nofollow">printer-friendly version</a> of this TotT episode and post it in your office.</span></i><br /> <i><br /></i><span style="font-family: "times" , "times new roman" , serif;">By Ben Yu</span><br /> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span style="font-family: "times" , "times new roman" , serif;"><span style="color: purple; font-weight: bold;"> Helper methods make it easier to create test data. But they can become difficult to read over time</span> as you need more variations of the test data to satisfy constantly evolving requirements from new tests:</span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #f4cccc; vertical-align: top; width: 607px;"><pre style="background-color: #f4cccc; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: #006600;">// This helper method starts with just a single parameter:</span> Company company = <span style="font-weight: bold;">newCompany</span>(PUBLIC);</span></pre> <pre style="background-color: #f4cccc; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;"> <span style="color: #006600;">// But soon it acquires more and more parameters. // Conditionals creep into the newCompany() method body to handle the nulls, // and the method calls become hard to read due to the long parameter lists: </span>Company small = <span style="font-weight: bold;">newCompany</span>(2, 2, null, PUBLIC); Company privatelyOwned = <span style="font-weight: bold;">newCompany</span>(null, null, null, PRIVATE); Company bankrupt = <span style="font-weight: bold;">newCompany</span>(null, null, PAST_DATE, PUBLIC); <span style="color: #006600;">// Or a new method is added each time a test needs a different combination of fields: </span>Company small = <span style="font-weight: bold;">newCompanyWithEmployeesAndBoardMembers</span>(2, 2, PUBLIC); Company privatelyOwned = <span style="font-weight: bold;">newCompanyWithType</span>(PRIVATE); Company bankrupt = <span style="font-weight: bold;">newCompanyWithBankruptcyDate</span>(PAST_DATE, PUBLIC);</span></pre> </td></tr> </tbody></table> </div> <span style="font-family: "times" , "times new roman" , serif;"><br /></span> <span style="font-family: "times" , "times new roman" , serif;">Instead, <span style="color: purple; font-weight: bold;">use the test data builder pattern: create a helper method that returns a partially-built object</span> (e.g., a Builder in languages such as Java, or a mutable object) whose state can be overridden in tests. The helper method initializes logically-required fields to reasonable defaults, so <span style="color: purple; font-weight: bold;">each test can specify only fields relevant to the case being tested</span>: </span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #d9ead3; vertical-align: top; width: 607px;"><pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;">Company small = <span style="font-weight: bold;">newCompany</span>().setEmployees(2).setBoardMembers(2).build(); Company privatelyOwned = <span style="font-weight: bold;">newCompany</span>().setType(PRIVATE).build(); Company bankrupt = <span style="font-weight: bold;">newCompany</span>().setBankruptcyDate(PAST_DATE).build(); Company arbitraryCompany = <span style="font-weight: bold;">newCompany</span>().build(); </span></pre> <pre style="background-color: #d9ead3; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: blue;"> </span></span></pre> <pre style="background-color: #d9ead3; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: blue;">// Zero parameters makes this method reusable for different variations of Company. // It also doesn’t need conditionals to ignore parameters that aren’t set (e.g. null // values) since a test can simply not set a field if it doesn’t care about it. </span>private static <span style="font-weight: bold;">Company.Builder newCompany</span>() { return Company.newBuilder().setType(PUBLIC).setEmployees(100); <span style="color: blue;">// Set required fields</span> }</span></pre> </td></tr> </tbody></table> </div> <br /> <span style="font-family: "times" , "times new roman" , serif;">Also note that <span style="color: purple; font-weight: bold;">tests should never rely on default values that are specified by a helper method</span> since that forces readers to read the helper method’s implementation details in order to understand the test.</span><br /> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr><td style="background-color: #cfe2f3; vertical-align: top; width: 607px;"><pre style="background-color: #cfe2f3; border: 0px; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="color: black; font-family: "consolas" , "courier new" , "courier" , monospace;"><span style="color: #006600;">// This test needs a public company, so explicitly set it. // It also needs a company with no board members, so explicitly clear it. </span>Company publicNoBoardMembers = newCompany().<span style="font-weight: bold;">setType(PUBLIC).clearBoardMembers</span>().build();</span></pre> </td></tr> </tbody></table> </div> <i><br /></i> <i><span style="font-family: "times" , "times new roman" , serif;">You can learn more about this topic at <a href="http://www.natpryce.com/articles/000714.html">http://www.natpryce.com/articles/000714.html</a></span></i> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Cleanly Create Test Data&url=https://testing.googleblog.com/2018/02/testing-on-toilet-cleanly-create-test.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2018/02/testing-on-toilet-cleanly-create-test.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2018/02/testing-on-toilet-cleanly-create-test.html#comments' style='font-weight: 500; text-decoration: underline;'>7 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2018/02/testing-on-toilet-cleanly-create-test.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Ben%20Yu' rel='tag'> Ben Yu </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='blog-pager' id='blog-pager'> <a class='home-link' href='https://testing.googleblog.com/'> <i class='material-icons'>  </i> </a> <span id='blog-pager-newer-link'> <a class='blog-pager-newer-link' href='https://testing.googleblog.com/search?updated-max=2020-07-16T09:10:00-07:00&max-results=5&reverse-paginate=true' id='Blog1_blog-pager-newer-link' title='Newer Posts'> <i class='material-icons'>  </i> </a> </span> <span id='blog-pager-older-link'> <a class='blog-pager-older-link' href='https://testing.googleblog.com/search?updated-max=2018-02-20T14:11:00-08:00&max-results=5' id='Blog1_blog-pager-older-link' title='Older Posts'> <i class='material-icons'>  </i> </a> </span> </div> <div class='clear'></div> </div></div> </div> </div> <div class='col-right'> <div class='section' id='sidebar-top'><div class='widget HTML' data-version='1' id='HTML8'> <div class='widget-content'> <div class='searchBox'> <input type='text' title='Search This Blog' placeholder='Search blog ...' /> </div> </div> <div class='clear'></div> </div> </div> <div id='aside'> <div class='section' id='sidebar'><div class='widget Label' data-version='1' id='Label1'> <div class='tab'> <img class='sidebar-icon' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII='/> <h2> Labels </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content list-label-widget-content'> <ul> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/TotT'> TotT </a> <span dir='ltr'> 102 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/GTAC'> GTAC </a> <span dir='ltr'> 61 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/James%20Whittaker'> James Whittaker </a> <span dir='ltr'> 42 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Misko%20Hevery'> Misko Hevery </a> <span dir='ltr'> 32 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Code%20Health'> Code Health </a> <span dir='ltr'> 30 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anthony%20Vallone'> Anthony Vallone </a> <span dir='ltr'> 27 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrick%20Copeland'> Patrick Copeland </a> <span dir='ltr'> 23 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jobs'> Jobs </a> <span dir='ltr'> 18 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Andrew%20Trenk'> Andrew Trenk </a> <span dir='ltr'> 12 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/C%2B%2B'> C++ </a> <span dir='ltr'> 11 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund'> Patrik Höglund </a> <span dir='ltr'> 8 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/JavaScript'> JavaScript </a> <span dir='ltr'> 7 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Allen%20Hutchison'> Allen Hutchison </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/George%20Pirocanac'> George Pirocanac </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zhanyong%20Wan'> Zhanyong Wan </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Harry%20Robinson'> Harry Robinson </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Java'> Java </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julian%20Harty'> Julian Harty </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Bender'> Adam Bender </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alberto%20Savoia'> Alberto Savoia </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ben%20Yu'> Ben Yu </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Erik%20Kuefler'> Erik Kuefler </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Philip%20Zembrod'> Philip Zembrod </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shyam%20Seshadri'> Shyam Seshadri </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome'> Chrome </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dillon%20Bly'> Dillon Bly </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Thomas'> John Thomas </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Lesley%20Katzen'> Lesley Katzen </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Kaplan'> Marc Kaplan </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Markus%20Clermont'> Markus Clermont </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander'> Max Kanat-Alexander </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sonal%20Shah'> Sonal Shah </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/APIs'> APIs </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Abhishek%20Arya'> Abhishek Arya </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Myrvold'> Alan Myrvold </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alek%20Icev'> Alek Icev </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Android'> Android </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/April%20Fools'> April Fools </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chaitali%20Narla'> Chaitali Narla </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chris%20Lewis'> Chris Lewis </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome%20OS'> Chrome OS </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Salas'> Diego Salas </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dori%20Reuveni'> Dori Reuveni </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Arbon'> Jason Arbon </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jochen%20Wuttke'> Jochen Wuttke </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kostya%20Serebryany'> Kostya Serebryany </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Eaddy'> Marc Eaddy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87'> Marko Ivanković </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mobile'> Mobile </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Oliver%20Chang'> Oliver Chang </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simon%20Stewart'> Simon Stewart </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stefan%20Kennedy'> Stefan Kennedy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Flakiness'> Test Flakiness </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Titus%20Winters'> Titus Winters </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tony%20Voellm'> Tony Voellm </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/WebRTC'> WebRTC </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yiming%20Sun'> Yiming Sun </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yvette%20Nameth'> Yvette Nameth </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zuri%20Kemp'> Zuri Kemp </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Aaron%20Jacobs'> Aaron Jacobs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Porter'> Adam Porter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Raider'> Adam Raider </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adel%20Saoud'> Adel Saoud </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Faulkner'> Alan Faulkner </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alex%20Eagle'> Alex Eagle </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Amy%20Fu'> Amy Fu </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anantha%20Keesara'> Anantha Keesara </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Antoine%20Picard'> Antoine Picard </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/App%20Engine'> App Engine </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ari%20Shamash'> Ari Shamash </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Arif%20Sukoco'> Arif Sukoco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Benjamin%20Pick'> Benjamin Pick </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bob%20Nystrom'> Bob Nystrom </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bruce%20Leban'> Bruce Leban </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Arguelles'> Carlos Arguelles </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Israel%20Ortiz%20Garc%C3%ADa'> Carlos Israel Ortiz García </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Cathal%20Weakliam'> Cathal Weakliam </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Christopher%20Semturs'> Christopher Semturs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Clay%20Murphy'> Clay Murphy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dagang%20Wei'> Dagang Wei </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Maksimovich'> Dan Maksimovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Shi'> Dan Shi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Willemsen'> Dan Willemsen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Chen'> Dave Chen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Gladfelter'> Dave Gladfelter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Bendory'> David Bendory </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Mandelberg'> David Mandelberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Derek%20Snyder'> Derek Snyder </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Cavalcanti'> Diego Cavalcanti </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov'> Dmitry Vyukov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz'> Eduardo Bravo Ortiz </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ekaterina%20Kamenskaya'> Ekaterina Kamenskaya </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliott%20Karpilovsky'> Elliott Karpilovsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliotte%20Rusty%20Harold'> Elliotte Rusty Harold </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Espresso'> Espresso </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Felipe%20Sodr%C3%A9'> Felipe Sodré </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Francois%20Aube'> Francois Aube </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Gene%20Volovich'> Gene Volovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Google%2B'> Google+ </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goran%20Petrovic'> Goran Petrovic </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goranka%20Bjedov'> Goranka Bjedov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hank%20Duan'> Hank Duan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Havard%20Rast%20Blok'> Havard Rast Blok </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hongfei%20Ding'> Hongfei Ding </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Elbaum'> Jason Elbaum </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Huggins'> Jason Huggins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jay%20Han'> Jay Han </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Hoy'> Jeff Hoy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Listfield'> Jeff Listfield </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jessica%20Tomechak'> Jessica Tomechak </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jim%20Reardon'> Jim Reardon </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joe%20Allan%20Muharsky'> Joe Allan Muharsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joel%20Hynoski'> Joel Hynoski </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Micco'> John Micco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Penix'> John Penix </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway'> Jonathan Rockway </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez'> Jonathan Velasquez </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Josh%20Armour'> Josh Armour </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julie%20Ralph'> Julie Ralph </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kai%20Kent'> Kai Kent </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Karin%20Lundberg'> Karin Lundberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kaue%20Silveira'> Kaue Silveira </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Bourrillion'> Kevin Bourrillion </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Graney'> Kevin Graney </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kirkland'> Kirkland </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever'> Kurt Alfred Kluever </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Manjusha%20Parvathaneni'> Manjusha Parvathaneni </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis'> Marek Kiszkis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marius%20Latinis'> Marius Latinis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Ivey'> Mark Ivey </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Manley'> Mark Manley </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Striebeck'> Mark Striebeck </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Matt%20Lowrie'> Matt Lowrie </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Meredith%20Whittaker'> Meredith Whittaker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Bachman'> Michael Bachman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Klepikov'> Michael Klepikov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Aizatsky'> Mike Aizatsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Wacker'> Mike Wacker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy'> Mona El Mahdy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Noel%20Yap'> Noel Yap </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Palak%20Bansal'> Palak Bansal </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patricia%20Legaspi'> Patricia Legaspi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Per%20Jacobsson'> Per Jacobsson </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht'> Peter Arrenbrecht </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Spragins'> Peter Spragins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Norman'> Phil Norman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Rollet'> Phil Rollet </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Pooja%20Gupta'> Pooja Gupta </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Project%20Showcase'> Project Showcase </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Radoslav%20Vasilev'> Radoslav Vasilev </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Dewan'> Rajat Dewan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Jain'> Rajat Jain </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rich%20Martin'> Rich Martin </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Richard%20Bustamante'> Richard Bustamante </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Roshan%20Sembacuttiaratchy'> Roshan Sembacuttiaratchy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov'> Ruslan Khamitov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sam%20Lee'> Sam Lee </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sean%20Jordan'> Sean Jordan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sharon%20Zhou'> Sharon Zhou </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shiva%20Garg'> Shiva Garg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Siddartha%20Janga'> Siddartha Janga </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simran%20Basi'> Simran Basi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stan%20Chan'> Stan Chan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stephen%20Ng'> Stephen Ng </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tejas%20Shah'> Tejas Shah </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Analytics'> Test Analytics </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Engineer'> Test Engineer </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tim%20Lyakhovetskiy'> Tim Lyakhovetskiy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tom%20O%27Neill'> Tom O'Neill </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Vojta%20J%C3%ADna'> Vojta Jína </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/automation'> automation </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/dead%20code'> dead code </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/iOS'> iOS </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/mutation%20testing'> mutation testing </a> <span dir='ltr'> 1 </span> </li> </ul> <div class='clear'></div> </div> </div><div class='widget BlogArchive' data-version='1' id='BlogArchive1'> <div class='tab'> <i class='material-icons icon'>  </i> <h2> Archive </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content'> <div id='ArchiveList'> <div id='BlogArchive1_ArchiveList'> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/'> 2024 </a> <span class='post-count' dir='ltr'>(12)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/'> 2023 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/09/'> Sep </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/'> 2022 </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/'> 2021 </a> <span class='post-count' dir='ltr'>(3)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/'> 2020 </a> <span class='post-count' dir='ltr'>(8)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/'> 2019 </a> <span class='post-count' dir='ltr'>(4)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/'> 2018 </a> <span class='post-count' dir='ltr'>(7)</span> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2018/11/testing-on-toilet-exercise-service-call.html'> Testing on the Toilet: Exercise Service Call Contr... </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2018/09/efficacy-presubmit.html'> Efficacy Presubmit </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2018/07/code-health-make-interfaces-hard-to.html'> Code Health: Make Interfaces Hard to Misuse </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2018/06/testing-on-toilet-only-verify-relevant.html'> Testing on the Toilet: Only Verify Relevant Method... </a> </li> <li> <a href='https://testing.googleblog.com/2018/06/testing-on-toilet-keep-tests-focused.html'> Testing on the Toilet: Keep Tests Focused </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html'> Code Health: Understanding Code In Review </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2018/02/testing-on-toilet-cleanly-create-test.html'> Testing on the Toilet: Cleanly Create Test Data </a> </li> </ul> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/'> 2017 </a> <span class='post-count' dir='ltr'>(17)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/'> 2016 </a> <span class='post-count' dir='ltr'>(15)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/'> 2015 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/'> 2014 </a> <span class='post-count' dir='ltr'>(24)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/06/'> Jun </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/'> 2013 </a> <span class='post-count' dir='ltr'>(16)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/'> 2012 </a> <span class='post-count' dir='ltr'>(11)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/'> 2011 </a> <span class='post-count' dir='ltr'>(39)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/06/'> Jun </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/05/'> May </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/01/'> Jan </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/'> 2010 </a> <span class='post-count' dir='ltr'>(37)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/11/'> Nov </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/10/'> Oct </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/03/'> Mar </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/'> 2009 </a> <span class='post-count' dir='ltr'>(54)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/07/'> Jul </a> <span class='post-count' dir='ltr'>(15)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/06/'> Jun </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/'> 2008 </a> <span class='post-count' dir='ltr'>(75)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/12/'> Dec </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/11/'> Nov </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/10/'> Oct </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/08/'> Aug </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/07/'> Jul </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/06/'> Jun </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/05/'> May </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/04/'> Apr </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/02/'> Feb </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/'> 2007 </a> <span class='post-count' dir='ltr'>(41)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/10/'> Oct </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/04/'> Apr </a> <span class='post-count' dir='ltr'>(7)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/03/'> Mar </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> </div> </div> <div class='clear'></div> </div> </div><div class='widget HTML' data-version='1' id='HTML6'> <div class='widget-content'> <a href="http://googletesting.blogspot.com/atom.xml"> <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==" class="sidebar-icon" /> <h2>Feed</h2> </a> </div> <div class='clear'></div> </div></div> <div class='section' id='sidebar-bottom'><div class='widget HTML' data-version='1' id='HTML9'> <div class='widget-content'> <a href='http://cloud.feedly.com/#subscription%2Ffeed%2Fhttp%3A%2F%2Fgoogletesting.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='blank'><img id="feedlyFollow" src="https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_vBbGZcE_pmOYRKoAbFgakY8S-HCUSVgB0utRK-0kXWxdtMxUa8wnWdBaVmSVQTDqeBxIRSdCo6GrnO4nqT6Nv1kIprK8KSQMUBC6w8RspSRwYg3p6Ct5lm6acA3leD3jO168YzuOytt9YGeF5umMI=s0-d" alt="follow us in feedly" width="66" height="20"></a> <div class="share followgooglewrapper"> <button data-href="https://twitter.com/intent/follow?original_referer=http://googletesting.blogspot.com/&screen_name=googletesting" onclick='sharingPopup(this);' id='twitter-share'><span class="twitter-follow">Follow @googletesting</span></button> <script> function sharingPopup (button) { var url = button.getAttribute("data-href"); window.open( url,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); } </script> </div> </div> <div class='clear'></div> </div></div> </div> </div> <div style='clear:both;'></div> </div>  <div class='google-footer-outer loading'> <div id='google-footer'> <a href='//www.google.com/'> <img class='google-logo-dark' height='36' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==' style='margin-top: -16px;' width='92'/> </a> <ul> <li> <a href='//www.google.com/'> Google </a> </li> <li> <a href='//www.google.com/policies/privacy/'> Privacy </a> </li> <li> <a href='//www.google.com/policies/terms/'> Terms </a> </li> </ul> </div> </div> <script type='text/javascript'> //<![CDATA[ // Social sharing popups. var postEl = document.getElementsByClassName('social-wrapper'); var postCount = postEl.length; for(i=0; i<postCount;i++){ postEl[i].addEventListener("click", function(event){ var postUrl = this.getAttribute("data-href"); window.open( postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); });} //]]> </script> <script type='text/javascript'> //<![CDATA[ var BreakpointHandler = function() { this.initted = false; this.isHomePage = false; this.isMobile = false; }; BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) { // Use $.trim for IE8 compatibility summaryHtml = $.trim(summaryHtml).replace(/(<br>|\s)+$/,''); if (lastNode.nodeType == 3) { var lastChar = summaryHtml.slice(-1); if (!lastChar.match(/[.”"?]/)) { if (!lastChar.match(/[A-Za-z]/)) { summaryHtml = summaryHtml.slice(0, -1); } summaryHtml += ' ...'; } } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) { summaryHtml += ' ...'; } return summaryHtml; }; BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) { var seenWords = 0; var summaryHtml = ''; for (var i=0; i < content.childNodes.length; i++) { var node = content.childNodes[i]; var nodeText; if (node.nodeType == 1) { if (node.hasAttribute('data-about-pullquote')) { continue; } nodeText = node.textContent; if (nodeText === undefined) { // innerText for IE8 nodeText = node.innerText; } if (node.nodeName == 'DIV' || node.nodeName == 'B') { // Don't end early if we haven't seen enough words. if (seenWords < 10) { continue; } if (i > 0) { summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]); } break; } summaryHtml += node.outerHTML; } else if (node.nodeType == 3) { nodeText = node.nodeValue; summaryHtml += nodeText + ' '; } var words = nodeText.match(/\S+\s*/g); if (!words) { continue; } var remain = numWords - seenWords; if (words.length >= remain) { summaryHtml = this.finalizeSummary(summaryHtml, node); break; } seenWords += words.length; } return summaryHtml; }; BreakpointHandler.prototype.detect = function() { var match, pl = /\+/g, search = /([^&=]+)=?([^&]*)/g, decode = function (s) { return decodeURIComponent(s.replace(pl, " ")); }, query = window.location.search.substring(1); var urlParams = {}; while (match = search.exec(query)) urlParams[decode(match[1])] = decode(match[2]); this.isListPage = $('html').hasClass('list-page'); this.isMobile = urlParams['m'] === '1'; this.isHomePage = window.location.pathname == '/'; }; BreakpointHandler.prototype.initContent = function() { var self = this; $('.post').each(function(index) { var body = $(this).children('.post-body')[0]; var content = $(body).children('.post-content')[0]; $(content).addClass('post-original'); var data = $(content).children('script').html(); data = self.rewriteForSSL(data); // If exists, extract specified editor's preview. var match = data.match(/([\s\S]+?)<div data-is-preview.+?>([\s\S]+)<\/div>/m); if (match) { data = match[1]; } // Prevent big images from loading when they aren't needed. // This must be done as a pre-injection step, since image loading can't be // canceled once embedded into the DOM. if (self.isListPage && self.isMobile) { data = data.replace(/<(img|iframe) .+?>/g, ''); } // Insert template to be rendered as nodes. content.innerHTML = data; if (self.isListPage) { var summary = document.createElement('div'); $(summary).addClass('post-content'); $(summary).addClass('post-summary'); body.insertBefore(summary, content); if (match) { // Use provided summary. summary.innerHTML = match[2]; } else { // Generate a summary. // Summary generation relies on DOM, so it must occur after content is // inserted into the page. summary.innerHTML = self.generateSummaryFromContent(content, 30); } // Add read more link to summary. var titleAnchor = $(this).find('.title a')[0]; var link = titleAnchor.cloneNode(true); link.innerHTML = 'Read More'; $(link).addClass('read-more'); summary.appendChild(link); } }); // Firefox does not allow for proper styling of BR. if (navigator.userAgent.indexOf('Firefox') > -1) { $('.post-content br').replaceWith('<span class="space"></span>'); } $('.loading').removeClass('loading'); }; BreakpointHandler.prototype.process = function() { if (!this.initted) { var makeInsecureImageRegex = function(hosts) { var whitelist = hosts.join('|').replace(/\./g,'\\.'); // Normal image tags, plus input images (yes, this is possible!) return new RegExp('(<(img|input)[^>]+?src=("|\'))http:\/\/(' + whitelist +')', 'g'); }; this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS); this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]); this.detect(); this.initContent(); this.initted = true; } }; BreakpointHandler.KNOWN_HTTPS_HOSTS = [ "www.google.org", "www.google.com", "services.google.com", "blogger.com", "draft.blogger.com", "www.blogger.com", "photos1.blogger.com", "photos2.blogger.com", "photos3.blogger.com", "blogblog.com", "img1.blogblog.com", "img2.blogblog.com", "www.blogblog.com", "www1.blogblog.com", "www2.blogblog.com", "0.bp.blogspot.com", "1.bp.blogspot.com", "2.bp.blogspot.com", "3.bp.blogspot.com", "4.bp.blogspot.com", "lh3.googleusercontent.com", "lh4.googleusercontent.com", "lh5.googleusercontent.com", "lh6.googleusercontent.com", "themes.googleusercontent.com", ]; BreakpointHandler.prototype.rewriteForSSL = function(html) { // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content. return html.replace(this.sslImageRegex, '$1https://$4') .replace(this.sslImageCurrentDomainRegex, '$1//$4') .replace(/(<(embed|iframe)[^>]+?src=("|'))http:\/\/([^"']*?(youtube|picasaweb\.google)\.com)/g, '$1https://$4') // Slideshow SWF takes a image host, so we need to rewrite that parameter. .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s'); }; $(document).ready(function() { var handler = new BreakpointHandler(); handler.process(); // Top-level navigation. $(".BlogArchive .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); $(".Label .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); // Blog archive year expansion. $('.BlogArchive .intervalToggle').click(function(ev) { ev.preventDefault(); if ($(this).parent().hasClass('collapsed')) { $(this).parent().removeClass('collapsed'); $(this).parent().addClass('expanded'); } else { $(this).parent().removeClass('expanded'); $(this).parent().addClass('collapsed'); } }); // Reverse order of months. $('.BlogArchive .intervalToggle + div').each(function(_, items) { var year = $(this); year.children().each(function(_, month) { year.prepend(month); }); }); // Set anchors to open in new tab. $('.post-content img').parent().each(function(_, node) { if (node.nodeName == 'A') { $(this).attr('target', '_blank'); } }); // Process search requests. $('.searchBox input').on("keypress", function(ev) { if (ev.which == 13) { window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val()); } }); }); (function($, window) { var archiveButton = $($('#sidebar .widget.BlogArchive h2')[0]); var folderIcon = $('#sidebar .widget.BlogArchive h2::after'); var archivePanel = $('#BlogArchive1'); archiveButton.click(function(e) { if (archivePanel.hasClass('archive-open')) { // It's open, so now we close it archivePanel.removeClass('archive-open'); archivePanel.addClass('archive-closed'); archiveButton.removeClass('archive-open'); archiveButton.addClass('archive-closed'); archivePanel.css('height', '60px'); } else { // It's closed, so open it archivePanel.removeClass('archive-closed'); archivePanel.addClass('archive-open'); archiveButton.removeClass('archive-closed'); archiveButton.addClass('archive-open'); archivePanel.css('height', 'auto'); folderIcon.css('content', 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEggkDzi8QVSua_XxLOnfxWE_8nQc1MAhRPBxJej91JW2ZBbKvb2TjaNf66ZoTqCrBivwRr-TLJYxFm2ZAiMd-zfbJ0TjblXGvUszWFbdRncNOL7jseiJCL9uQoqN8BM8FpesdSw/s1600/keyboard_arrow_up_grey600_24dp.png'); } }); })($, window); (function ($, window, devsite) { devsite.devsite = {}; devsite.expandLeftNav = devsite.expandLeftNav || {}; devsite.expandLeftNav.ANALYTICS_LABEL_ = 'Hamburger menu'; devsite.expandLeftNav.ANALYTICS_ACTION_OPEN_ = 'Open'; devsite.expandLeftNav.ANALYTICS_ACTION_CLOSE_ = 'Close'; devsite.expandLeftNav.lastState_ = 0; devsite.expandLeftNav.chekovEnabled_ = false; devsite.expandLeftNav.init = function () { devsite.expandLeftNav.chekovEnabled_ = true; var navElement; if (devsite.expandLeftNav.chekovEnabled_) { navElement = $('.devsite-nav-responsive'); } else { navElement = $('.devsite-section-nav-responsive'); } this.drawerWidth = parseInt(navElement.css('width')); navElement.css({ 'left': -(this.drawerWidth) }); var expandButton = $('.devsite-expand-section-nav'); expandButton.click(function () { devsite.expandLeftNav.handleNavigationOpened(); }); navElement.find('.devsite-nav-responsive-forward').click( devsite.expandLeftNav.openPanel); navElement.find('.devsite-nav-responsive-back').click( devsite.expandLeftNav.closePanel); if ($('.devsite-nav-responsive-tabs-panel + ' + '.devsite-nav-responsive-sidebar-panel').length) { devsite.expandLeftNav.openPanel(); } }; devsite.expandLeftNav.handleNavigationOpened = function (opt_noAnimate) { var mask; if (devsite.expandLeftNav.chekovEnabled_) { var nav = $('.devsite-nav-responsive'); if (opt_noAnimate) { nav.addClass('devsite-nav-responsive-no-animate'); mask = devsite.devsite.showSiteMask(0); } else { mask = devsite.devsite.showSiteMask(); } nav.addClass('devsite-nav-responsive-open'); } else { devsite.expandLeftNav.lastState_ = devsite.sticky.currentState; devsite.sticky.setState(devsite.sticky.state.COLLAPSED_HEADER); var headerHeight = devsite.sticky.getHeaderHeight() - devsite.sticky.desiredMargin; var drawerHeight = $(window).height() - headerHeight; mask = devsite.devsite.showArticleMask(); $('.devsite-section-nav-responsive').css({ 'height': drawerHeight, 'left': '0', 'top': headerHeight, 'visibility': 'visible' }).focus(); } $(mask).click(devsite.expandLeftNav.handleNavigationClosed); $(document).on('keydown.escape', function (e) { if (e.keyCode == $.ui.keyCode.ESCAPE) { devsite.expandLeftNav.handleNavigationClosed(); $(document).off('keydown.escape'); } }); }; devsite.expandLeftNav.getScrollTop = function () { return $(window).scrollTop(); }; devsite.expandLeftNav.handleNavigationClosed = function () { var nav; if (devsite.expandLeftNav.chekovEnabled_) { nav = $('.devsite-nav-responsive'); devsite.devsite.hideSiteMask(); nav.removeClass('devsite-nav-responsive-open ' + 'devsite-nav-responsive-no-animate'); } else { nav = $('.devsite-section-nav-responsive'); devsite.devsite.hideArticleMask(); devsite.sticky.setState(devsite.expandLeftNav.lastState_); nav .css('left', -(devsite.expandLeftNav.drawerWidth)) .one('transitionend', function () { nav.css('visibility', 'hidden'); }); } nav.scrollTop(0); }; devsite.expandLeftNav.openPanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel.show(); parentPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel .removeClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); parentPanel.hide(); }); setTimeout(function () { childPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); }, 1); }; devsite.expandLeftNav.closePanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); parentPanel .show() .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); setTimeout(function () { parentPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); childPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel.hide(); parentPanel.removeClass('devsite-nav-responsive-transition'); }); }, 1); }; devsite.expandLeftNav.FADE_SLOW_ = 'slow'; devsite.expandLeftNav.FADE_FAST_ = 'fast'; devsite.expandLeftNav.SITE_MASK_CSS_ = '.devsite-site-mask'; devsite.devsite.showSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_SLOW_; } devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, false, opt_animate); }; devsite.devsite.hideSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_FAST_; } devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, true, opt_animate); }; devsite.devsite.showArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_('.devsite-article-mask', false, devsite.expandLeftNav.FADE_SLOW_); }; devsite.devsite.hideArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_('.devsite-article-mask', true, devsite.expandLeftNav.FADE_FAST_); }; devsite.devsite.setMask_ = function(className, out, opt_fadeTime) { var query = $(className); if (opt_fadeTime === 0) { out ? query.hide() : query.show(); } else { out ? query.fadeOut(opt_fadeTime) : query.fadeIn(opt_fadeTime); } return $(className)[0]; }; devsite.devsite.setMouseScrollingEnabled = function(trueOrFalse) { if (trueOrFalse == true) { $('html, body').css({ 'overflow': '' }); } else { $('html, body').css({ 'overflow': 'hidden' }); } }; })($, window, devsite = {}); if (window.jQuery) { $(document).ready(function () { if (window.devsite) { devsite.expandLeftNav.init(); } }); } //]]> </script> <style> .widget ul{ line-height: 1.6 !important; } #sidebar ul li a{ color:black; line-height: 20px; } #sidebar ul li a:hover{ color:#4184F3; } </style> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY7kcIyDhiUhaNJoRHTqdDj6z5R9nw:1732448984161';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d15045980','//testing.googleblog.com/2018/','15045980'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '15045980', 'title': 'Google Testing Blog', 'url': 'https://testing.googleblog.com/2018/', 'canonicalUrl': 'https://testing.googleblog.com/2018/', 'homepageUrl': 'https://testing.googleblog.com/', 'searchUrl': 'https://testing.googleblog.com/search', 'canonicalHomepageUrl': 'https://testing.googleblog.com/', 'blogspotFaviconUrl': 'https://testing.googleblog.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'G-838ZCPQWM6', 'analytics4': true, 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Google Testing Blog - RSS\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/15045980/posts/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/d78375fb222d99b3', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'archive', 'pageName': '2018', 'pageTitle': 'Google Testing Blog: 2018'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Google Testing Blog', 'description': '', 'url': 'https://testing.googleblog.com/2018/', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': true, 'isLabelSearch': false, 'archive': {'year': 2018, 'rangeMessage': 'Showing posts from 2018'}}}]); _WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/2646514562-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML9', 'sidebar-bottom', document.getElementById('HTML9'), {}, 'displayModeFull')); </script> </body> </html>