CINXE.COM

<!DOCTYPE html> <html class='v2 list-page' dir='ltr' itemscope='' itemtype='http://schema.org/Blog' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <title> Google Testing Blog: mutation testing </title> <meta content='width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0' name='viewport'/> <meta content='IE=Edge' http-equiv='X-UA-Compatible'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='en_US' property='og:locale'/> <meta content='https://testing.googleblog.com/search/label/mutation%20testing' property='og:url'/> <meta content='Google Testing Blog' property='og:site_name'/>  <meta content='Google Testing Blog' property='og:title'/> <meta content='summary' name='twitter:card'/> <meta content='@googletesting' name='twitter:creator'/> <link href='https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic' rel='stylesheet' type='text/css'/> <link href='https://fonts.googleapis.com/icon?family=Material+Icons' rel='stylesheet'/> <script src='https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js' type='text/javascript'></script>  <style id='page-skin-1' type='text/css'></style> <style id='template-skin-1' type='text/css'></style>  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://testing.googleblog.com/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://testing.googleblog.com/search/label/mutation%20testing' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://testing.googleblog.com/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="Google Testing Blog - RSS" href="https://testing.googleblog.com/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://www.blogger.com/feeds/15045980/posts/default" />  <meta content='https://testing.googleblog.com/search/label/mutation%20testing' property='og:url'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='' property='og:description'/>  <base target='_self'/> <style> html { font-family: Roboto, sans-serif; -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; } body { padding: 0; /* This ensures that the scroll bar is always present, which is needed */ /* because content render happens after page load; otherwise the header */ /* would "bounce" in-between states. */ min-height: 150%; } h2 { font-size: 20px; } h1, h2, h3, h4, h5 { line-height: 2em; } html, h4, h5, h6 { font-size: 17px; } h3 { font-size: 18px !important; } a, a:visited { color: #4184F3; text-decoration: none; } a:focus, a:hover, a:active { text-decoration: none; } .Header { margin-top: 15px; } .Header h1 { font-size: 32px; font-weight: 300; line-height: 32px; height: 42px; } .header-inner .Header .titlewrapper { padding: 0; margin-top: 30px; } .header-inner .Header .descriptionwrapper { padding: 0; margin: 0; } .cols-wrapper { margin-top: 56px; } .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 60px; } .header-inner { height: 256px; position: relative; } html, .header-inner a { color: #212121; color: rgba(0,0,0,.87); } .header-inner .google-logo { display: inline-block; background-size: contain; z-index: 1; height: 46px; overflow: hidden; margin-top: 4px; margin-right: 8px; } .header-left { position: absolute; top: 50%; -webkit-transform: translateY(-50%); transform: translateY(-50%); margin-top: 12px; width: 100%; } .google-logo { margin-left: -4px; } #google-footer { position: relative; font-size: 13px; list-style: none; text-align: right; } #google-footer a { color: #444; } #google-footer ul { margin: 0; padding: 0; height: 144px; line-height: 144px; } #google-footer ul li { display: inline; } #google-footer ul li:before { color: #999; content: "\00b7"; font-weight: bold; margin: 5px; } #google-footer ul li:first-child:before { content: ''; } #google-footer .google-logo-dark { left: 0; margin-top: -16px; position: absolute; top: 50%; } /** Sitemap links. **/ .footer-inner-2 { font-size: 14px; padding-top: 42px; padding-bottom: 74px; } .footer-inner-2 .HTML h2 { color: #212121; color: rgba(0,0,0,.87); font-size: 14px; font-weight: 500; padding-left: 0; margin: 10px 0; } .footer-inner-2 .HTML ul { font-weight: normal; list-style: none; padding-left: 0; } .footer-inner-2 .HTML li { line-height: 24px; padding: 0; } .footer-inner-2 li a { color: rgba(65,132,243,.87); } /** Archive widget. **/ .BlogArchive { font-size: 13px; font-weight: normal; } .BlogArchive .widget-content { display: none; } .BlogArchive h2, .Label h2 { color: #4184F3; text-decoration: none; } .BlogArchive .hierarchy li { display: inline-block; } /* Specificity needed here to override widget CSS defaults. */ .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li { padding-left: 0; text-indent: 0; } .BlogArchive .intervalToggle { cursor: pointer; } .BlogArchive .expanded .intervalToggle .new-toggle { -ms-transform: rotate(180deg); transform: rotate(180deg); } .BlogArchive .new-toggle { float: right; padding-top: 3px; opacity: 0.87; } #ArchiveList { text-transform: uppercase; } #ArchiveList .expanded > ul:last-child { margin-bottom: 16px; } #ArchiveList .archivedate { width: 100%; } /* Months */ .BlogArchive .items { max-width: 150px; margin-left: -4px; } .BlogArchive .expanded .items { margin-bottom: 10px; overflow: hidden; } .BlogArchive .items > ul { float: left; height: 32px; } .BlogArchive .items a { padding: 0 4px; } .Label { font-size: 13px; font-weight: normal; } .sidebar-icon { display: inline-block; width: 24px; height: 24px; vertical-align: middle; margin-right: 12px; margin-top: -1px } .Label a { margin-right: 4px; } .Label .widget-content { display: none; } .FollowByEmail { font-size: 13px; font-weight: normal; } .FollowByEmail h2 { background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); background-repeat: no-repeat; background-position: 0 50%; text-indent: 30px; } .FollowByEmail .widget-content { display: none; } .searchBox input { border: 1px solid #eee; color: #212121; color: rgba(0,0,0,.87); font-size: 14px; padding: 8px 8px 8px 40px; width: 164px; font-family: Roboto, sans-serif; background: url("https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png") 8px center no-repeat; } .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */ color: rgba(0,0,0,.54); } .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */ color: #000; opacity: 0.54; } .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */ color: #000; opacity: 0.54; } .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */ color: #757575; } .widget-item-control { margin-top: 0px; } .section { margin: 0; padding: 0; } #sidebar-top { border: 1px solid #eee; } #sidebar-top > div { margin: 16px 0; } .widget ul { line-height: 1.6; } /*main post*/ .post { margin-bottom:30px; } #main .post .title { margin: 0; } #main .post .title a { color: #212121; color: rgba(0,0,0,.87); font-weight: normal; font-size: 24px; } #main .post .title a:hover { text-decoration:none; color:#4184F3; } .message, #main .post .post-header { margin: 0; padding: 0; } #main .post .post-header .caption, #main .post .post-header .labels-caption, #main .post .post-footer .caption, #main .post .post-footer .labels-caption { color: #444; font-weight: 500; } #main .tr-caption-container td { text-align: left; } #main .post .tr-caption { color: #757575; color: rgba(0,0,0,.54); display: block; max-width: 560px; padding-bottom: 20px; } #main .post .tr-caption-container { line-height: 24px; margin: -1px 0 0 0 !important; padding: 4px 0; text-align: left; } #main .post .post-header .published{ font-size:11px; font-weight:bold; } .post-header .publishdate { font-size: 17px; font-weight:normal; color: #757575; color: rgba(0,0,0,.54); } #main .post .post-footer{ font-size:12px; padding-bottom: 21px; } .label-footer { margin-bottom: 12px; margin-top: 12px; } .comment-img { margin-right: 16px; opacity: 0.54; vertical-align: middle; } #main .post .post-header .published { margin-bottom: 40px; margin-top: -2px; } .post .post-content { color: #212121; color: rgba(0,0,0,.87); font-size: 17px; margin: 25px 0 36px 0; line-height: 32px; word-wrap: break-word; } .post-body .post-content ul, .post-body .post-content ol { margin: 16px 0; padding: 0 48px; } .post-summary { display: none; } /* Capture section headers. */ .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b { display: inline-block; margin-bottom: 8px; margin-top: 24px; } .post-content li { line-height: 32px; } /* Override all post images/videos to left align. */ .post-content .separator, .post-content > div { text-align: left; } .post-content .separator > a, .post-content .separator > span { margin-left: 0 !important; } .post-content img { max-width: 100%; } .post-content .tr-caption-container img { margin-bottom: 12px; } .post-content iframe, .post-content embed { max-width: 100%; } .post-content .carousel-container { margin-bottom: 48px; } #main .post-content b { font-weight: 500; } /* These are the main paragraph spacing tweaks. */ #main .post-content br { content: ' '; display: block; padding: 4px; } .post-content .space { display: block; height: 8px; } .post-content iframe + .space, .post-content iframe + br { padding: 0 !important; } #main .post .jump-link { margin-bottom:10px; } .post-content img, .post-content iframe { margin: 30px 0 20px 0; } .post-content > img:first-child, .post-content > iframe:first-child { margin-top: 0; } .col-right .section { padding: 0 16px; } #aside { background:#fff; border:1px solid #eee; border-top: 0; } #aside .widget { margin:0; } #aside .widget h2, #ArchiveList .toggle + a.post-count-link { color: #212121; color: rgba(0,0,0,.87); font-weight: 400 !important; margin: 0; } #ArchiveList .toggle { float: right; } #ArchiveList .toggle .material-icons { padding-top: 4px; } #sidebar .tab { cursor: pointer; } #sidebar .tab .arrow { display: inline-block; float: right; } #sidebar .tab .icon { display: inline-block; vertical-align: top; height: 24px; width: 24px; margin-right: 13px; margin-left: -1px; margin-top: 1px; color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget-content > :first-child { padding-top: 8px; } #sidebar .active .tab .arrow { -ms-transform: rotate(180deg); transform: rotate(180deg); } #sidebar .arrow { color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget h2 { font-size: 14px; line-height: 24px; display: inline-block; } #sidebar .widget .BlogArchive { padding-bottom: 8px; } #sidebar .widget { border-bottom: 1px solid #eee; box-shadow: 0px 1px 0 white; margin-bottom: 0; padding: 14px 0; min-height: 20px; } #sidebar .widget:last-child { border-bottom: none; box-shadow: none; margin-bottom: 0; } #sidebar ul { margin: 0; padding: 0; } #sidebar ul li { list-style:none; padding:0; } #sidebar ul li a { line-height: 32px; } #sidebar .archive { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC"); height: 24px; line-height: 24px; padding-left: 30px; } #sidebar .labels { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC"); height: 20px; line-height: 20px; padding-left: 30px; } #sidebar .rss a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII="); } #sidebar .subscription a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); } #sidebar-bottom { background: #f5f5f5; border-top:1px solid #eee; } #sidebar-bottom .widget { border-bottom: 1px solid #e0e0e0; padding: 15px 0; text-align: center; } #sidebar-bottom > div:last-child { border-bottom: 0; } #sidebar-bottom .text { line-height: 20px; } /* Home, forward, and backward pagination. */ .blog-pager { border-top : 1px #e0e0e0 solid; padding-top: 10px; margin-top: 15px; text-align: right !important; } #blog-pager { margin-botom: 0; margin-top: -14px; padding: 16px 0 0 0; } #blog-pager a { display: inline-block; } .blog-pager i.disabled { opacity: 0.2 !important; } .blog-pager i { color: black; margin-left: 16px; opacity: 0.54; } .blog-pager i:hover, .blog-pager i:active { opacity: 0.87; } #blog-pager-older-link, #blog-pager-newer-link { float: none; } .gplus-profile { background-color: #fafafa; border: 1px solid #eee; overflow: hidden; width: 212px; } .gplus-profile-inner { margin-left: -1px; margin-top: -1px; } /* Sidebar follow buttons. */ .followgooglewrapper { padding: 12px 0 0 0; } .loading { visibility: hidden; } .detail-page .post-footer .cmt_iframe_holder { padding-top: 40px !important; } /** Desktop **/ @media (max-width: 900px) { .col-right { display: none; } .col-main { margin-right: 0; min-width: initial; } .footer-outer { display: none; } .cols-wrapper { min-width: initial; } .google-footer-outer { background-color: #f5f5f5; } } /** Tablet **/ @media (max-width: 712px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 40px; } } /* An extra breakpoint accommodating for long blog titles. */ @media (max-width: 600px) { .header-left { height: 100%; position: initial; top: inherit; margin-top: 0; -webkit-transform: initial; transform: initial; } .header-title { margin-top: 18px; } .header-inner { height: auto; margin-bottom: 32px; margin-top: 32px; } .header-desc { margin-top: 12px; } .header-inner .google-logo { height: 40px; margin-top: 3px; } .header-inner .google-logo img { height: 42px; } .header-title h2 { font-size: 32px; line-height: 40px; } } /** Mobile/small desktop window; also landscape. **/ @media (max-width: 480px), (max-height: 480px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 16px; } .cols-wrapper { margin-top: 0; } .post-header .publishdate, .post .post-content { font-size: 16px; } .post .post-content { line-height: 28px; margin-bottom: 30px; } .post { margin-top: 30px; } .byline-author { display: block; font-size: 12px; line-height: 24px; margin-top: 6px; } #main .post .title a { font-weight: 500; color: #4c4c4c; color: rgba(0,0,0,.70); } #main .post .post-header { padding-bottom: 12px; } #main .post .post-header .published { margin-bottom: -8px; margin-top: 3px; } .post .read-more { display: block; margin-top: 14px; } .post .tr-caption { font-size: 12px; } #main .post .title a { font-size: 20px; line-height: 30px; } .post-content iframe { /* iframe won't keep aspect ratio when scaled down. */ max-height: 240px; } .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe { margin-left: -16px; max-width: inherit; width: calc(100% + 32px); } .post-content table, .post-content td { width: 100%; } #blog-pager { margin: 0; padding: 16px 0; } /** List page tweaks. **/ .list-page .post-original { display: none; } .list-page .post-summary { display: block; } .list-page .comment-container { display: none; } .list-page #blog-pager { padding-top: 0; border: 0; margin-top: -8px; } .list-page .label-footer { display: none; } .list-page #main .post .post-footer { border-bottom: 1px solid #eee; margin: -16px 0 0 0; padding: 0 0 20px 0; } .list-page .post .share { display: none; } /** Detail page tweaks. **/ .detail-page .post-footer .cmt_iframe_holder { padding-top: 32px !important; } .detail-page .label-footer { margin-bottom: 0; } .detail-page #main .post .post-footer { padding-bottom: 0; } .detail-page #comments { display: none; } } [data-about-pullquote], [data-is-preview], [data-about-syndication] { display: none; } </style> <noscript> <style> .loading { visibility: visible }</style> </noscript>  <script async='true' src='https://www.googletagmanager.com/gtag/js?id=G-838ZCPQWM6'></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-838ZCPQWM6'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <script type='text/javascript'> //<![CDATA[ var axel = Math.random() + ""; var a = axel * 10000000000000; document.write('<iframe src="https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?" width="1" height="1" frameborder="0" style="display:none"></iframe>'); //]]> </script> <noscript> <img alt='' height='1' src='https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?' width='1'/> </noscript>  <div class='header-outer'> <div class='header-inner'> <div class='section' id='header'><div class='widget Header' data-version='1' id='Header1'> <div class='header-left'> <div class='header-title'> <a class='google-logo' href='https://testing.googleblog.com/'> <img height='50' src='https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png'/> </a> <a href='/.'> <h2> Testing Blog </h2> </a> </div> <div class='header-desc'> </div> </div> </div></div> </div> </div>  <div class='cols-wrapper loading'> <div class='col-main-wrapper'> <div class='col-main'> <div class='section' id='main'><div class='widget Blog' data-version='1' id='Blog1'> <div class='post' data-id='3647813866660315609' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2021/04/mutation-testing.html' itemprop='url' title='Mutation Testing'> Mutation Testing </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Monday, April 12, 2021 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <style> .code-reserved { color: #9c27b0; } .code-method, .code-attribute { color: #3367d6; } .code-string { color: #0f9d58; } .code-numerical { color: #c53929; } .code-changed-added { background-color: #d9ead3; border: 1px solid green; } .code-changed-removed { background-color: #f4cccc; border: 1px solid red; } </style> By Goran Petrovic<div><br /></div><h2 style="text-align: left;">History </h2><div><br /></div><div>It’s been a long-standing tradition of my team to organize hackathons twice a year. In weeks prior to the hackathon, the team gathers and brainstorms ideas for projects, ranging from improving the testing infrastructure or an existing process, to trying out a wild idea they’ve had for some time. Just before the hackathon, the team rates the accumulated ideas on a coolness-impact scale: how much fun does a project sound vs. how impactful could it potentially be; while impact is important, for hackathons, fun is non-negotiable. Then, engineers who are excited to work on some of the proposed projects subscribe and form teams. It was no different in the cold winter of 2013, where among the plethora of cool and wild ideas, one was to prototype <b>Mutation testing.</b></div><div><br /></div><div><br /></div><div>For those who are not familiar with it, mutation testing is a method of evaluating test quality by injecting bugs into the code and seeing whether the tests detect the fault or not. The more injected bugs the tests catch, the better they are. Here’s an example:</div><div><br /></div><div><br /></div><div>Negating the <i>if</i> condition.</div><div><br /></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">def</span> checkout(cart): <span class="code-reserved">if <span class="code-changed-added">not</span></span> cart.items: <span class="code-reserved">throw</span> <span class="code-method">Error</span>(<span class="code-string">"cart empty"</span>) <span class="code-reserved">return</span> checkout_internal(cart)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">def</span> checkout(cart): <span class="code-reserved">if</span> cart.items: <span class="code-reserved">throw</span> <span class="code-method">Error</span>(<span class="code-string">"cart empty"</span>) <span class="code-reserved">return</span> checkout_internal(cart)</span></pre></td> </tr> </tbody></table><br /> </div> If a test fails, we say it kills the mutant, and if no tests fail, we say that the mutant is alive.<div><br /></div><div><br /></div><div><div>By the end of the hackathon, mutagenesis was implemented for C++ and Python, and a prototype was born: a shell script that evaluates generated mutants in a diff (pull request) and textually reports live mutants in the files in the diff. A year passed with no work done on the project, before I started to work on it in my <a href="https://en.wikipedia.org/wiki/20%25_Project">20% time</a>. I had no idea what Mutation testing was at the time, so I researched and read papers on the topic, and collected lots of ideas on what I should focus on.</div></div><div><br /></div><div><br /></div><div><h2>From Prototype To Launch</h2></div><div><br /></div><div><div>I quickly realized that the hackathon crew did not calculate the Mutation score, the ratio of mutants detected by tests, which is a prominent metric in the research literature and the holy grail of evaluating test quality, but just enumerated live mutants. My first exposure to mutants was just running the tool on the mutagenesis code itself and trying to understand the report. I was immediately overwhelmed: after a long execution time, I was facing thousands of mutants in just a handful of files. I tried going through a couple, but after a few minutes I grew tired and moved on with my main project, which happened to be on Google Shopping. In the following months, I stayed away from my 20% project, but I kept thinking about it, bugging my colleagues and friends about the ideas I had to make mutation testing work for us. After many months of brainstorming and discussions, almost a year after the original hackathon project, I was ready to design the Mutation Testing Service.</div></div><div><br /></div><div><br /></div><div><div>I faced two big problems. First, I could force myself to go through lots of mutants, and maybe find a useful one that would prompt me to write a test case, but I could not force others, not even my teammates. Second, the vast majority of mutants were simply bad. Here are some examples:</div></div> <div><br /></div><div><br /></div><div>Replacing division with subtraction, but in a logging statement.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">log.<span class="code-method">Infof</span>(<span class="code-string">"Found %d (%.2f %%)!"</span>, e, float64(e)*<span class="code-numerical">100.0</span> <span class="code-changed-removed">/</span> total)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">log.<span class="code-method">Infof</span>(<span class="code-string">"Found %d (%.2f %%)!"</span>, e, float64(e)*<span class="code-numerical">100.0</span> <span class="code-changed-added">-</span> total)</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Appending number 1 to an error message. </div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-method">Error</span>.create(((key + <span class="code-string">" disabled"</span>)));</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-method">Error</span>.create(((key + <span class="code-string">" disabled"</span>) <span class="code-changed-added">+ <span class="code-numerical">1</span></span>));</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div><div>Replacing <i>greater than with less than</i> when comparing length of a collection to zero.</div></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">showCart := len(cart.GetItems()) <span class="code-changed-removed">></span> <span class="code-numerical">0</span></span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">showCart := len(cart.GetItems()) <span class="code-changed-added"><</span> <span class="code-numerical">0</span></span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div><div>Replacing the idiomatic python check for whether the module is imported or executed as main.</div></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> (__name__ <span class="code-changed-removed">==</span> <span class="code-string">'__main__'</span>):</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> (__name__ <span class="code-changed-added">!=</span> <span class="code-string">'__main__'</span>):</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Changing python’s string concatenation (+) to string multiplication (*).</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">message = (<span class="code-string">'id '</span> <span class="code-changed-removed">+</span> run_idx)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">message = (<span class="code-string">'id '</span> <span class="code-changed-added">*</span> run_idx)</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Changing a tuning parameter.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">slo = (<span class="code-numerical">20</span> * time.<span class="code-attribute">Second</span>)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">slo = (<span class="code-numerical">20</span> * time.<span class="code-attribute">Second</span>) <span class="code-changed-added">+ <span class="code-numerical">1</span></span></span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Changing a network timeout, but the network layer is mocked in the tests.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">_TIMEOUT = (<span class="code-numerical">60</span> <span class="code-changed-removed">*</span> <span class="code-numerical">10</span>)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">_TIMEOUT = (<span class="code-numerical">60</span> <span class="code-changed-added">/</span> <span class="code-numerical">10</span>)</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Subtracting from -∞.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">df = df.replace( [numpy.inf, -numpy.inf], numpy.nan )</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">df = df.replace( [numpy.inf, -numpy.inf <span class="code-changed-added">- <span class="code-numerical">1</span></span>], numpy.nan )</span></pre></td> </tr> </tbody></table><br /> </div><br /><div>Yes, the tests did not detect these mutants, but we would not want such tests anyway. Many of them would produce fragile, <a href="https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html">change-detector tests</a>. We later settled on calling them <b>unproductive mutants</b>: writing tests for those mutants would make the test suite worse, not better.</div><div><br /></div> <div><br /></div><div><div>I realized that I needed to suppress these types of mutants: if I reported them, nobody would use mutation testing, myself included. Most of the mutants were not useful, and that is a waste of developer attention. The onus was on me to create a better tool. I set out to try various heuristics by looking at the report and suppressing mutants that I found unproductive. I encoded the heuristics in AST (<a href="https://en.wikipedia.org/wiki/Abstract_syntax_tree">Abstract Syntax Tree</a>) matching rules, and I dubbed the AST nodes which contained unproductive mutants as <b>arid nodes</b>. In the beginning, there were only a few rules in the system, but that was enough to make me feel confident that my colleagues would give it a try.</div></div><div><br /></div><div><br /></div><div><div>The other big issue was the sheer number of mutants. With five or more in a line, hundreds in a file, it was a challenge to display them, and even if I managed that, nobody would go through them anyway. I quickly realized that they shouldn’t: it took a lot of time for me to go through the mutants, and, while some pointed me to a hole in my test suite, most were useless, and many of them, especially ones in the same line, redundant. I did not need every possible combination of operators changed to tell me that my test for that condition was insufficient; one was just fine. That was my first decision on mutation testing: to report at most one mutant in a line. This was a quick and easy decision to make, because, if you’ve ever used a Code review system, you know that having more makes the review noisy and harder to do. Another reason why it was such an easy decision was that it would have been computationally prohibitively expensive to calculate all mutants, and I could have thrown my 20% project down the drain. I call it limitation-driven development :)</div></div><div><br /></div><div><br /></div><div>Of course, the idea was to report live mutants during <a href="https://research.google/pubs/pub47025/">Code review</a>. Code review is the perfect time in the engineering process to surface useful findings about the code being changed, and integrating into the existing developer process has the <a href="https://research.google/pubs/pub43477/">highest chance</a> that the developers will take action. This seemed like the most normal thing in the world: we had hundreds of <a href="https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43322.pdf">analyzers</a> and all engineers were used to receiving findings from various analyses of their code. It took an outsider to point out that this was a strange approach: mutation testing was classically run on the whole program and the mutation score calculated and used as guidance.</div><div><br /></div><div><br /></div><div>This is what a Mutant finding looks like in the Code review tool:</div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjFQqkssBFxzWfHfdT51aECi83loojgVOQ-Xh3sTPS9qkkJLHoifI3SuZ-98ftCx4i4gn8xZD7dIyM0b9bI3lJipRuUK2ZkCbihyUkRRK6EYjCZ98j4dpMyR93jUhw0_cGD_i4H/s607/Code+Review+Image+for+Mutation+Testing+Article.jpg" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="430" data-original-width="607" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjFQqkssBFxzWfHfdT51aECi83loojgVOQ-Xh3sTPS9qkkJLHoifI3SuZ-98ftCx4i4gn8xZD7dIyM0b9bI3lJipRuUK2ZkCbihyUkRRK6EYjCZ98j4dpMyR93jUhw0_cGD_i4H/s16000/Code+Review+Image+for+Mutation+Testing+Article.jpg" /></a></div><div><br /></div><div><br /></div><div><br /></div>Mutation Testing at Google is a dynamic analyzer of code changes that surfaces mutants during Code review by posting code findings. In terms of infrastructure, it consists of three main parts: the <i>change listener</i>, the <i>analyzer</i>, and many <i>mutagenesis servers</i>, one for each language.<br /><div><br /></div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh816h-dDe5XtCUKDTyd4xzIkKP5Gl94y9oZ-viuEUD5aMWv1av60qQoRvGEninqYS23aVu7t1p4Za5jgtzdSp0tPKexoQronquB9NXY0itBZt8z3FUGTWeCYFlwu3f39baX3dr/s711/Block+Diagram+for+Mutation+Testing+Article.jpg" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="466" data-original-width="711" height="421" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh816h-dDe5XtCUKDTyd4xzIkKP5Gl94y9oZ-viuEUD5aMWv1av60qQoRvGEninqYS23aVu7t1p4Za5jgtzdSp0tPKexoQronquB9NXY0itBZt8z3FUGTWeCYFlwu3f39baX3dr/w640-h421/Block+Diagram+for+Mutation+Testing+Article.jpg" width="640" /></a></div><br /><div><br /></div><div><br /></div><div><div>Each event during the Code review is announced using a publisher-subscriber pattern, and any interested party can listen, and react, to these events. When a change is sent for Code review, many things happen: linters are run, automated tests are evaluated, coverage is calculated, and for the users of mutation testing, mutants are generated and evaluated. Listening on all events coming from the Code review system, the <i>listener</i> schedules a mutation run on the <i>analyzer</i>. </div></div><div><br /></div><div><br /></div><div>The first thing the analyzer does is get the code coverage results for the patch in question; from it, the analyzer can extrapolate which tests cover which lines of source code. This is a very useful piece of information, because running the minimum set of tests that can kill a mutant is crucial; if we just ran all tests that were linked in, or covered the project, that would be prohibitively computationally expensive</div><div><br /></div><div><br /></div><div><div>Next, for each covered line in each file in the patch, a <i>mutagenesis server</i> for the language in question is asked to produce a mutant. The mutagenesis server parses the file, traverses its AST, and applies the mutators in the requested order (as per mutation context), ignoring arid nodes, nodes in uncovered lines and in lines that are not affected by the proposed patch.</div></div><div><br /></div><div><br /></div><div><div>When the <i>analyzer</i> assembles all mutants, it patches them one by one to a version control context and then evaluates all the tests for each mutant in parallel. For mutants for which all tests pass, the <i>analyzer</i> surfaces a finding for the code author and reviewers, and is done for the time being.</div></div><div><br /></div><div><br /></div><div>Because the Code review is a laborious and dynamic process, with many rounds of comments from reviewers and many automated findings from hundreds of different analyzers, there can be many snapshots as the patch evolves: adoption of recommendations from reviewers or accepting proposed changes from linters yields many code states. Mutation testing first runs after coverage is available, and then it runs for each subsequent snapshot: developers like to see the fruits of their labor: when they write a test to kill a mutant, they want to see the mutant killed.</div><div><br /></div><div><br /></div><div>I launched Mutation testing for the Shopping Engineering Productivity team in late 2015. Around 15 of my colleagues were subjected to Mutant findings during their Code reviews, and it was a bumpy start. Each finding has two buttons: <b>Please fix</b> and <b>Not useful</b>, as you can see on the Code review screenshot above. A reviewer can instruct the code author to fix some finding (e.g. a ClangTidy finding might point out that an object is being unnecessarily copied and suggest using a reference instead, or a Mutant finding might point out that code is not well tested). The author and all reviewers can give feedback to the author of the finding/analyzer that their finding is not useful. This is a source of valuable information, and I made use of it. For each mutant that was deemed not useful, I’d check it out and see whether I could generalize from it and add a new rule to my arid node heuristics. Slowly, I collected several hundred heuristics, many of them generally applicable, but many also tied to internal stuff, like monitoring frameworks. More and more, I noticed that just marking nodes as arid and suppressing mutants in them was not enough on its own; a more powerful mechanism was required to reduce this noise even further. Take a look at these motivating examples:</div> <div><br /></div><div>Changing the condition of an if statement, but the body is arid (a logging statement).</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> _, err := <span class="code-method">Del</span>(req); err <span class="code-changed-removed">!=</span> <span class="code-reserved">nil</span> { log.<span class="code-method">Errorf</span>(<span class="code-string">"cleanup failed: %v”</span>, cerr) }</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> _, err := c.<span class="code-method">Del</span>(req); err <span class="code-changed-added">==</span> <span class="code-reserved">nil</span> { log.<span class="code-method">Errorf</span>(<span class="code-string">"cleanup failed: %v”</span>, cerr) }</span></pre></td> </tr> </tbody></table><br /> </div> <br /><div><br /></div><div><div>Similar pattern, but in C++:</div></div><div><br /></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> (<span class="code-changed-added">!</span>(!status.ok())) { LOG(WARNING) << <span class="code-string">"Updating dependency graph failed"</span> << status; }</span></pre></td> </tr> </tbody></table><br /> </div><br /><div><div>I settled for a transitive rule: an AST node is arid if I say it’s arid, or if it’s a compound statement and all its body is also arid. This made sense in retrospect, but it took some looking at reported examples of unproductive mutants to crystalize. Because the logging statements are arid, the whole <i>if</i> statement’s body is arid, and hence, the if statement itself is arid, including its condition. </div></div><div><br /></div><div><br /></div><div><div>In the summer of 2015, my intern, Małgorzata Salawa, and I got mutagenesis implemented for C++, Go, Python, and Java, and having transitive arid node detection and surfacing at most a single mutant per line and 7 per file, we called it a v1.0 and launched. Mutation testing was always an opt-in service, and in the beginning had a few users (93 code reviews in Q1 of 2016), but over time it ramped up to 2,500 users in February 2017, to tens of thousands today. The early days were crucial to get the users’ feedback and extend the arid node heuristics ever further. In the beginning, the <i>Not Useful</i> rate was around 80%, and this was already with some heuristics and at most a single mutant per line. With time, I got it down to around 15%. I was always aware that getting the rate to 0% was impossible, because of the nature of the mutants: sometimes, the mutant would produce an equivalent behavior as the original, and there was no way to avoid that fully.</div></div><div><br /></div><div><br /></div> <div>Changing cached lookup by removing the cache and always recalculating yields functionally equivalent code, undetectable by conventional testing.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">func (s *Serv) calculate(<span class="code-reserved">in</span> int32) <span class="code-reserved">int</span> {<span class="code-changed-added"> <span class="code-reserved">if</span> val, ok := <span class="code-reserved">if</span> s.cache[<span class="code-reserved">in</span>] { <span class="code-reserved">return</span> val }</span> val := s.calc(<span class="code-reserved">in</span>) s.cache[<span class="code-reserved">in</span>] = val <span class="code-reserved">return</span> val }</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">func (s *Serv) calculate(<span class="code-reserved">in</span> int32) <span class="code-reserved">int</span> {<span class="code-changed-removed"> </span> val := s.calc(<span class="code-reserved">in</span>) s.cache[<span class="code-reserved">in</span>] = val <span class="code-reserved">return</span> val }</span></pre></td> </tr> </tbody></table></div><br /><div><br /></div><div><br /></div><div><div>I was both surprised and happy that I could lower the <i>Not useful</i> rate to under 50%.</div></div><div><br /></div><div><br /></div><div><h2>Mutation Context</h2></div><div><br /></div><div>As time went by, I added support for more languages. In early 2017, I implemented support for JavaScript and TypeScript, and later in the year I added support for Dart. In 2018 I added support for <a href="https://github.com/google/zetasql">ZetaSQL</a>. And finally, in early 2020, I added support for Kotlin as it became more and more popular in the Android world.</div><div><br /></div><div><br /></div><div><div>I kept track of various stats for all mutants: their survival rates and <i>Please fix/Not useful</i> ratios. </div></div><div><br /></div><div><br /></div><div>The worst performing mutator was ABS(<i>Absolute Value Mutator</i>) that would replace an <i>expression</i> with <i>±abs(expression),</i> for example:</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">absl::<span class="code-method">Minutes</span>(<span class="code-numerical">10</span>) - elapsed;</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">absl::<span class="code-method">Minutes</span>(<span class="code-changed-added">-abs</span>(<span class="code-numerical">10</span>)) - elapsed;</span></pre></td> </tr> </tbody></table><br /> </div><div style="overflow-x: auto;"><br /></div><div style="overflow-x: auto;">Looking at the examples, I had to agree. Because the feedback was predominantly negative for this mutator, I quickly completely disabled it for all languages.</div><div style="overflow-x: auto;"><br /></div><div style="overflow-x: auto;"><br /></div><div style="overflow-x: auto;"><div style="overflow-x: auto;">I soon noticed that the SBR (<i>Statement Block Removal</i>) mutator, which deletes statements or whole blocks, is the most common one, and that made sense: while mutating a logical or arithmetic operator required the existence of such an operator in the code to be mutated, any line of code was eligible for deletion. Mutants generated by code deletion, though, did not have the best reported usefulness, or productivity. In fact, almost all other mutators generated more productive mutants than the SBR, and that got me thinking: not all code is the same; a condition within an <i>if</i> statement that contains a <i>return</i> statement is not the same as a condition in another location.</div><div><br /></div><div><br /></div><div><div>Out of this, an idea was born: context-based mutator selection. For each line, I would randomly shuffle mutator operators and pick one by one until one generated a mutant in that line. That was not ideal, because I knew that some operators worked better than others in some contexts. Rather than just randomly picking a mutant operator, I decided to pick the one most likely to generate a surviving mutant that is then most likely to be productive when reported, based on historical data. I had millions of mutants to learn from, I just needed to define the distance between pieces of code. Finally deciding to look for AST nodes that were in similar contexts as the node being mutated, I looked at the nodes around the node under mutation, and encoded the child-parent relationships of the near-by nodes to capture the AST context. Armed with the distance measure and with the help of my returning intern Małgorzata, it was easy to find the closest AST contexts from historic mutants and to look at their fate and pick the best one. I ordered the mutators by their productivity and tried to generate a mutant in the node, in that order, since it’s quite possible that some of the mutant operators are not applicable on some piece of code.</div><div><br /></div></div><div><br /></div><div>This was quite an improvement. Both mutant survivability and usefulness improved significantly for all mutant operators and programming languages. You can read more about the findings in <a href="https://arxiv.org/pdf/2102.11378.pdf">the upcoming paper</a>.</div><div><br /></div><div><br /></div><div><div><h2>Fault Coupling</h2></div><div><br /></div></div><div><br /></div><div>Mutation testing is only valuable if the test cases we write for mutants are valuable. <a href="https://ieeexplore.ieee.org/document/6982626" rel="nofollow" target="_blank">Mutants do not resemble real bugs</a>, they are simpler than bugs found in the wild. Mutation testing relies on the coupling hypothesis: <a href="https://dl.acm.org/doi/abs/10.1145/2635868.2635929" rel="nofollow" target="_blank">mutants are coupled with real bugs</a> if a test suite that is sensitive enough to detect mutants is also sensitive enough to detect the more complex real bugs. Reporting mutants and writing tests to kill them only makes sense if they are coupled with real bugs.</div><div><br /></div><div><br /></div><div><br /></div><div><div>I instinctively thought that fault coupling was real, otherwise I would not have worked on mutation testing at all, and I’ve seen many many cases where mutants pointed to a bug; but still, I wanted to verify this hypothesis, if only for our code base. I designed an experiment: I would generate all mutants in a line for explicit bug-fixing changes, before and after the fix, and I would check whether, had mutation testing been run, it would have surfaced a mutant in the change that introduced the bug, and potentially prevented it (i.e., killed in the change that fixed the bug and added new tests cases). I ran this experiment on weekends for over a month, because we did not have the resources to run it during workdays. While I normally generate a single mutant in a line, to test the fault coupling effect, I used the classical mutation testing approach and generated all possible mutants, while still adhering to the arid node suppression. A total of 33 million test suites were executed to test hundreds of thousands of mutants, finally to conclude that, in around 70% of cases, a bug was coupled with a mutant. </div></div><div><br /></div><div><br /></div><div>While I was at it, I also checked my intuition on whether a single mutant per line was enough, and found that it was overwhelmingly so: in more than 90% of cases, either all mutants were killed in a line or none was. It’s worth keeping in mind that I still applied my arid node suppression heuristics for this experiment. It was great to finally have confirmation of my intuitions.</div><div><br /></div><div><br /></div><div>I also looked into the developer behavior changes after using mutation testing on a project for longer periods of time, and discovered that projects that use mutation testing get more tests over time, as developers get exposed to more and more mutants. Not only do developers write more test cases, but those test cases are more effective in killing mutants: less and less mutants get reported over time. I noticed this from personal experience too: when writing unit tests, I would see where I cut some corners in the tests, and anticipated the mutant. Now I just add the missing test cases, rather than facing a mutant in my Code review, and I rarely see mutants these days, as I’ve learned to anticipate and preempt them.</div><div><br /></div><div><br /></div><div>You can read more about the findings in our <a href="https://homes.cs.washington.edu/~rjust/publ/mutation_testing_practices_icse_2021.pdf">ICSE paper</a>.</div><div><br /></div><div><br /></div><div><h2>Conclusion</h2></div><div><br /></div><div><br /></div><div><div>It’s been a long road since that hackathon in the winter of 2013. Mutation testing was a lot of fun to work on. It had its challenges, and I had days where I thought that I would throw everything down the drain (I’m looking at you, clang), but I am glad I stuck with it.</div></div><div><br /></div><div><br /></div><div><br /></div><div><div>The most interesting part of the project was getting Mutation testing to scale to such a large code base, and that required redefining the problem and adapting it to the existing ecosystem that engineers were already used to. Another interesting angle was working, and learning from, the academic community, in particular Gordon Fraser (University of Passau) and René Just (University of Washington).</div></div><div><br /></div><div><br /></div><div><br /></div><div>I would like to encourage everyone to give one of the many open source mutation testing tools a try on their projects. With some tweaks here and there, it can be a great way to keep your software well tested.</div><div><br /></div><div><br /></div><div><br /></div></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <style> .code-reserved { color: #9c27b0; } .code-method, .code-attribute { color: #3367d6; } .code-string { color: #0f9d58; } .code-numerical { color: #c53929; } .code-changed-added { background-color: #d9ead3; border: 1px solid green; } .code-changed-removed { background-color: #f4cccc; border: 1px solid red; } </style> By Goran Petrovic<div><br /></div><h2 style="text-align: left;">History </h2><div><br /></div><div>It’s been a long-standing tradition of my team to organize hackathons twice a year. In weeks prior to the hackathon, the team gathers and brainstorms ideas for projects, ranging from improving the testing infrastructure or an existing process, to trying out a wild idea they’ve had for some time. Just before the hackathon, the team rates the accumulated ideas on a coolness-impact scale: how much fun does a project sound vs. how impactful could it potentially be; while impact is important, for hackathons, fun is non-negotiable. Then, engineers who are excited to work on some of the proposed projects subscribe and form teams. It was no different in the cold winter of 2013, where among the plethora of cool and wild ideas, one was to prototype <b>Mutation testing.</b></div><div><br /></div><div><br /></div><div>For those who are not familiar with it, mutation testing is a method of evaluating test quality by injecting bugs into the code and seeing whether the tests detect the fault or not. The more injected bugs the tests catch, the better they are. Here’s an example:</div><div><br /></div><div><br /></div><div>Negating the <i>if</i> condition.</div><div><br /></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">def</span> checkout(cart): <span class="code-reserved">if <span class="code-changed-added">not</span></span> cart.items: <span class="code-reserved">throw</span> <span class="code-method">Error</span>(<span class="code-string">"cart empty"</span>) <span class="code-reserved">return</span> checkout_internal(cart)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">def</span> checkout(cart): <span class="code-reserved">if</span> cart.items: <span class="code-reserved">throw</span> <span class="code-method">Error</span>(<span class="code-string">"cart empty"</span>) <span class="code-reserved">return</span> checkout_internal(cart)</span></pre></td> </tr> </tbody></table><br /> </div> If a test fails, we say it kills the mutant, and if no tests fail, we say that the mutant is alive.<div><br /></div><div><br /></div><div><div>By the end of the hackathon, mutagenesis was implemented for C++ and Python, and a prototype was born: a shell script that evaluates generated mutants in a diff (pull request) and textually reports live mutants in the files in the diff. A year passed with no work done on the project, before I started to work on it in my <a href="https://en.wikipedia.org/wiki/20%25_Project">20% time</a>. I had no idea what Mutation testing was at the time, so I researched and read papers on the topic, and collected lots of ideas on what I should focus on.</div></div><div><br /></div><div><br /></div><div><h2>From Prototype To Launch</h2></div><div><br /></div><div><div>I quickly realized that the hackathon crew did not calculate the Mutation score, the ratio of mutants detected by tests, which is a prominent metric in the research literature and the holy grail of evaluating test quality, but just enumerated live mutants. My first exposure to mutants was just running the tool on the mutagenesis code itself and trying to understand the report. I was immediately overwhelmed: after a long execution time, I was facing thousands of mutants in just a handful of files. I tried going through a couple, but after a few minutes I grew tired and moved on with my main project, which happened to be on Google Shopping. In the following months, I stayed away from my 20% project, but I kept thinking about it, bugging my colleagues and friends about the ideas I had to make mutation testing work for us. After many months of brainstorming and discussions, almost a year after the original hackathon project, I was ready to design the Mutation Testing Service.</div></div><div><br /></div><div><br /></div><div><div>I faced two big problems. First, I could force myself to go through lots of mutants, and maybe find a useful one that would prompt me to write a test case, but I could not force others, not even my teammates. Second, the vast majority of mutants were simply bad. Here are some examples:</div></div> <div><br /></div><div><br /></div><div>Replacing division with subtraction, but in a logging statement.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">log.<span class="code-method">Infof</span>(<span class="code-string">"Found %d (%.2f %%)!"</span>, e, float64(e)*<span class="code-numerical">100.0</span> <span class="code-changed-removed">/</span> total)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">log.<span class="code-method">Infof</span>(<span class="code-string">"Found %d (%.2f %%)!"</span>, e, float64(e)*<span class="code-numerical">100.0</span> <span class="code-changed-added">-</span> total)</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Appending number 1 to an error message. </div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-method">Error</span>.create(((key + <span class="code-string">" disabled"</span>)));</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-method">Error</span>.create(((key + <span class="code-string">" disabled"</span>) <span class="code-changed-added">+ <span class="code-numerical">1</span></span>));</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div><div>Replacing <i>greater than with less than</i> when comparing length of a collection to zero.</div></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">showCart := len(cart.GetItems()) <span class="code-changed-removed">></span> <span class="code-numerical">0</span></span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">showCart := len(cart.GetItems()) <span class="code-changed-added"><</span> <span class="code-numerical">0</span></span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div><div>Replacing the idiomatic python check for whether the module is imported or executed as main.</div></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> (__name__ <span class="code-changed-removed">==</span> <span class="code-string">'__main__'</span>):</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> (__name__ <span class="code-changed-added">!=</span> <span class="code-string">'__main__'</span>):</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Changing python’s string concatenation (+) to string multiplication (*).</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">message = (<span class="code-string">'id '</span> <span class="code-changed-removed">+</span> run_idx)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">message = (<span class="code-string">'id '</span> <span class="code-changed-added">*</span> run_idx)</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Changing a tuning parameter.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">slo = (<span class="code-numerical">20</span> * time.<span class="code-attribute">Second</span>)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">slo = (<span class="code-numerical">20</span> * time.<span class="code-attribute">Second</span>) <span class="code-changed-added">+ <span class="code-numerical">1</span></span></span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Changing a network timeout, but the network layer is mocked in the tests.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">_TIMEOUT = (<span class="code-numerical">60</span> <span class="code-changed-removed">*</span> <span class="code-numerical">10</span>)</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">_TIMEOUT = (<span class="code-numerical">60</span> <span class="code-changed-added">/</span> <span class="code-numerical">10</span>)</span></pre></td> </tr> </tbody></table><br /> </div> <div><br /></div><div><br /></div><div>Subtracting from -∞.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">df = df.replace( [numpy.inf, -numpy.inf], numpy.nan )</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">df = df.replace( [numpy.inf, -numpy.inf <span class="code-changed-added">- <span class="code-numerical">1</span></span>], numpy.nan )</span></pre></td> </tr> </tbody></table><br /> </div><br /><div>Yes, the tests did not detect these mutants, but we would not want such tests anyway. Many of them would produce fragile, <a href="https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html">change-detector tests</a>. We later settled on calling them <b>unproductive mutants</b>: writing tests for those mutants would make the test suite worse, not better.</div><div><br /></div> <div><br /></div><div><div>I realized that I needed to suppress these types of mutants: if I reported them, nobody would use mutation testing, myself included. Most of the mutants were not useful, and that is a waste of developer attention. The onus was on me to create a better tool. I set out to try various heuristics by looking at the report and suppressing mutants that I found unproductive. I encoded the heuristics in AST (<a href="https://en.wikipedia.org/wiki/Abstract_syntax_tree">Abstract Syntax Tree</a>) matching rules, and I dubbed the AST nodes which contained unproductive mutants as <b>arid nodes</b>. In the beginning, there were only a few rules in the system, but that was enough to make me feel confident that my colleagues would give it a try.</div></div><div><br /></div><div><br /></div><div><div>The other big issue was the sheer number of mutants. With five or more in a line, hundreds in a file, it was a challenge to display them, and even if I managed that, nobody would go through them anyway. I quickly realized that they shouldn’t: it took a lot of time for me to go through the mutants, and, while some pointed me to a hole in my test suite, most were useless, and many of them, especially ones in the same line, redundant. I did not need every possible combination of operators changed to tell me that my test for that condition was insufficient; one was just fine. That was my first decision on mutation testing: to report at most one mutant in a line. This was a quick and easy decision to make, because, if you’ve ever used a Code review system, you know that having more makes the review noisy and harder to do. Another reason why it was such an easy decision was that it would have been computationally prohibitively expensive to calculate all mutants, and I could have thrown my 20% project down the drain. I call it limitation-driven development :)</div></div><div><br /></div><div><br /></div><div>Of course, the idea was to report live mutants during <a href="https://research.google/pubs/pub47025/">Code review</a>. Code review is the perfect time in the engineering process to surface useful findings about the code being changed, and integrating into the existing developer process has the <a href="https://research.google/pubs/pub43477/">highest chance</a> that the developers will take action. This seemed like the most normal thing in the world: we had hundreds of <a href="https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43322.pdf">analyzers</a> and all engineers were used to receiving findings from various analyses of their code. It took an outsider to point out that this was a strange approach: mutation testing was classically run on the whole program and the mutation score calculated and used as guidance.</div><div><br /></div><div><br /></div><div>This is what a Mutant finding looks like in the Code review tool:</div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjFQqkssBFxzWfHfdT51aECi83loojgVOQ-Xh3sTPS9qkkJLHoifI3SuZ-98ftCx4i4gn8xZD7dIyM0b9bI3lJipRuUK2ZkCbihyUkRRK6EYjCZ98j4dpMyR93jUhw0_cGD_i4H/s607/Code+Review+Image+for+Mutation+Testing+Article.jpg" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="430" data-original-width="607" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjFQqkssBFxzWfHfdT51aECi83loojgVOQ-Xh3sTPS9qkkJLHoifI3SuZ-98ftCx4i4gn8xZD7dIyM0b9bI3lJipRuUK2ZkCbihyUkRRK6EYjCZ98j4dpMyR93jUhw0_cGD_i4H/s16000/Code+Review+Image+for+Mutation+Testing+Article.jpg" /></a></div><div><br /></div><div><br /></div><div><br /></div>Mutation Testing at Google is a dynamic analyzer of code changes that surfaces mutants during Code review by posting code findings. In terms of infrastructure, it consists of three main parts: the <i>change listener</i>, the <i>analyzer</i>, and many <i>mutagenesis servers</i>, one for each language.<br /><div><br /></div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh816h-dDe5XtCUKDTyd4xzIkKP5Gl94y9oZ-viuEUD5aMWv1av60qQoRvGEninqYS23aVu7t1p4Za5jgtzdSp0tPKexoQronquB9NXY0itBZt8z3FUGTWeCYFlwu3f39baX3dr/s711/Block+Diagram+for+Mutation+Testing+Article.jpg" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="466" data-original-width="711" height="421" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh816h-dDe5XtCUKDTyd4xzIkKP5Gl94y9oZ-viuEUD5aMWv1av60qQoRvGEninqYS23aVu7t1p4Za5jgtzdSp0tPKexoQronquB9NXY0itBZt8z3FUGTWeCYFlwu3f39baX3dr/w640-h421/Block+Diagram+for+Mutation+Testing+Article.jpg" width="640" /></a></div><br /><div><br /></div><div><br /></div><div><div>Each event during the Code review is announced using a publisher-subscriber pattern, and any interested party can listen, and react, to these events. When a change is sent for Code review, many things happen: linters are run, automated tests are evaluated, coverage is calculated, and for the users of mutation testing, mutants are generated and evaluated. Listening on all events coming from the Code review system, the <i>listener</i> schedules a mutation run on the <i>analyzer</i>. </div></div><div><br /></div><div><br /></div><div>The first thing the analyzer does is get the code coverage results for the patch in question; from it, the analyzer can extrapolate which tests cover which lines of source code. This is a very useful piece of information, because running the minimum set of tests that can kill a mutant is crucial; if we just ran all tests that were linked in, or covered the project, that would be prohibitively computationally expensive</div><div><br /></div><div><br /></div><div><div>Next, for each covered line in each file in the patch, a <i>mutagenesis server</i> for the language in question is asked to produce a mutant. The mutagenesis server parses the file, traverses its AST, and applies the mutators in the requested order (as per mutation context), ignoring arid nodes, nodes in uncovered lines and in lines that are not affected by the proposed patch.</div></div><div><br /></div><div><br /></div><div><div>When the <i>analyzer</i> assembles all mutants, it patches them one by one to a version control context and then evaluates all the tests for each mutant in parallel. For mutants for which all tests pass, the <i>analyzer</i> surfaces a finding for the code author and reviewers, and is done for the time being.</div></div><div><br /></div><div><br /></div><div>Because the Code review is a laborious and dynamic process, with many rounds of comments from reviewers and many automated findings from hundreds of different analyzers, there can be many snapshots as the patch evolves: adoption of recommendations from reviewers or accepting proposed changes from linters yields many code states. Mutation testing first runs after coverage is available, and then it runs for each subsequent snapshot: developers like to see the fruits of their labor: when they write a test to kill a mutant, they want to see the mutant killed.</div><div><br /></div><div><br /></div><div>I launched Mutation testing for the Shopping Engineering Productivity team in late 2015. Around 15 of my colleagues were subjected to Mutant findings during their Code reviews, and it was a bumpy start. Each finding has two buttons: <b>Please fix</b> and <b>Not useful</b>, as you can see on the Code review screenshot above. A reviewer can instruct the code author to fix some finding (e.g. a ClangTidy finding might point out that an object is being unnecessarily copied and suggest using a reference instead, or a Mutant finding might point out that code is not well tested). The author and all reviewers can give feedback to the author of the finding/analyzer that their finding is not useful. This is a source of valuable information, and I made use of it. For each mutant that was deemed not useful, I’d check it out and see whether I could generalize from it and add a new rule to my arid node heuristics. Slowly, I collected several hundred heuristics, many of them generally applicable, but many also tied to internal stuff, like monitoring frameworks. More and more, I noticed that just marking nodes as arid and suppressing mutants in them was not enough on its own; a more powerful mechanism was required to reduce this noise even further. Take a look at these motivating examples:</div> <div><br /></div><div>Changing the condition of an if statement, but the body is arid (a logging statement).</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> _, err := <span class="code-method">Del</span>(req); err <span class="code-changed-removed">!=</span> <span class="code-reserved">nil</span> { log.<span class="code-method">Errorf</span>(<span class="code-string">"cleanup failed: %v”</span>, cerr) }</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> _, err := c.<span class="code-method">Del</span>(req); err <span class="code-changed-added">==</span> <span class="code-reserved">nil</span> { log.<span class="code-method">Errorf</span>(<span class="code-string">"cleanup failed: %v”</span>, cerr) }</span></pre></td> </tr> </tbody></table><br /> </div> <br /><div><br /></div><div><div>Similar pattern, but in C++:</div></div><div><br /></div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;"><span class="code-reserved">if</span> (<span class="code-changed-added">!</span>(!status.ok())) { LOG(WARNING) << <span class="code-string">"Updating dependency graph failed"</span> << status; }</span></pre></td> </tr> </tbody></table><br /> </div><br /><div><div>I settled for a transitive rule: an AST node is arid if I say it’s arid, or if it’s a compound statement and all its body is also arid. This made sense in retrospect, but it took some looking at reported examples of unproductive mutants to crystalize. Because the logging statements are arid, the whole <i>if</i> statement’s body is arid, and hence, the if statement itself is arid, including its condition. </div></div><div><br /></div><div><br /></div><div><div>In the summer of 2015, my intern, Małgorzata Salawa, and I got mutagenesis implemented for C++, Go, Python, and Java, and having transitive arid node detection and surfacing at most a single mutant per line and 7 per file, we called it a v1.0 and launched. Mutation testing was always an opt-in service, and in the beginning had a few users (93 code reviews in Q1 of 2016), but over time it ramped up to 2,500 users in February 2017, to tens of thousands today. The early days were crucial to get the users’ feedback and extend the arid node heuristics ever further. In the beginning, the <i>Not Useful</i> rate was around 80%, and this was already with some heuristics and at most a single mutant per line. With time, I got it down to around 15%. I was always aware that getting the rate to 0% was impossible, because of the nature of the mutants: sometimes, the mutant would produce an equivalent behavior as the original, and there was no way to avoid that fully.</div></div><div><br /></div><div><br /></div> <div>Changing cached lookup by removing the cache and always recalculating yields functionally equivalent code, undetectable by conventional testing.</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">func (s *Serv) calculate(<span class="code-reserved">in</span> int32) <span class="code-reserved">int</span> {<span class="code-changed-added"> <span class="code-reserved">if</span> val, ok := <span class="code-reserved">if</span> s.cache[<span class="code-reserved">in</span>] { <span class="code-reserved">return</span> val }</span> val := s.calc(<span class="code-reserved">in</span>) s.cache[<span class="code-reserved">in</span>] = val <span class="code-reserved">return</span> val }</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">func (s *Serv) calculate(<span class="code-reserved">in</span> int32) <span class="code-reserved">int</span> {<span class="code-changed-removed"> </span> val := s.calc(<span class="code-reserved">in</span>) s.cache[<span class="code-reserved">in</span>] = val <span class="code-reserved">return</span> val }</span></pre></td> </tr> </tbody></table></div><br /><div><br /></div><div><br /></div><div><div>I was both surprised and happy that I could lower the <i>Not useful</i> rate to under 50%.</div></div><div><br /></div><div><br /></div><div><h2>Mutation Context</h2></div><div><br /></div><div>As time went by, I added support for more languages. In early 2017, I implemented support for JavaScript and TypeScript, and later in the year I added support for Dart. In 2018 I added support for <a href="https://github.com/google/zetasql">ZetaSQL</a>. And finally, in early 2020, I added support for Kotlin as it became more and more popular in the Android world.</div><div><br /></div><div><br /></div><div><div>I kept track of various stats for all mutants: their survival rates and <i>Please fix/Not useful</i> ratios. </div></div><div><br /></div><div><br /></div><div>The worst performing mutator was ABS(<i>Absolute Value Mutator</i>) that would replace an <i>expression</i> with <i>±abs(expression),</i> for example:</div> <div style="overflow-x: auto;"> <table class="my-bordered-table" style="width: 613px;"><tbody> <tr> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">absl::<span class="code-method">Minutes</span>(<span class="code-numerical">10</span>) - elapsed;</span></pre></td> <td style="background-color: #e0e0e0; vertical-align: top; width: 607px;"><pre style="background-color: #e0e0e0; border: 0px; color: black; margin: 0px; padding-bottom: 0px; padding-left: 0px; padding-top: 0px;"><span style="font-family: consolas, "courier new", courier, monospace;">absl::<span class="code-method">Minutes</span>(<span class="code-changed-added">-abs</span>(<span class="code-numerical">10</span>)) - elapsed;</span></pre></td> </tr> </tbody></table><br /> </div><div style="overflow-x: auto;"><br /></div><div style="overflow-x: auto;">Looking at the examples, I had to agree. Because the feedback was predominantly negative for this mutator, I quickly completely disabled it for all languages.</div><div style="overflow-x: auto;"><br /></div><div style="overflow-x: auto;"><br /></div><div style="overflow-x: auto;"><div style="overflow-x: auto;">I soon noticed that the SBR (<i>Statement Block Removal</i>) mutator, which deletes statements or whole blocks, is the most common one, and that made sense: while mutating a logical or arithmetic operator required the existence of such an operator in the code to be mutated, any line of code was eligible for deletion. Mutants generated by code deletion, though, did not have the best reported usefulness, or productivity. In fact, almost all other mutators generated more productive mutants than the SBR, and that got me thinking: not all code is the same; a condition within an <i>if</i> statement that contains a <i>return</i> statement is not the same as a condition in another location.</div><div><br /></div><div><br /></div><div><div>Out of this, an idea was born: context-based mutator selection. For each line, I would randomly shuffle mutator operators and pick one by one until one generated a mutant in that line. That was not ideal, because I knew that some operators worked better than others in some contexts. Rather than just randomly picking a mutant operator, I decided to pick the one most likely to generate a surviving mutant that is then most likely to be productive when reported, based on historical data. I had millions of mutants to learn from, I just needed to define the distance between pieces of code. Finally deciding to look for AST nodes that were in similar contexts as the node being mutated, I looked at the nodes around the node under mutation, and encoded the child-parent relationships of the near-by nodes to capture the AST context. Armed with the distance measure and with the help of my returning intern Małgorzata, it was easy to find the closest AST contexts from historic mutants and to look at their fate and pick the best one. I ordered the mutators by their productivity and tried to generate a mutant in the node, in that order, since it’s quite possible that some of the mutant operators are not applicable on some piece of code.</div><div><br /></div></div><div><br /></div><div>This was quite an improvement. Both mutant survivability and usefulness improved significantly for all mutant operators and programming languages. You can read more about the findings in <a href="https://arxiv.org/pdf/2102.11378.pdf">the upcoming paper</a>.</div><div><br /></div><div><br /></div><div><div><h2>Fault Coupling</h2></div><div><br /></div></div><div><br /></div><div>Mutation testing is only valuable if the test cases we write for mutants are valuable. <a href="https://ieeexplore.ieee.org/document/6982626" rel="nofollow" target="_blank">Mutants do not resemble real bugs</a>, they are simpler than bugs found in the wild. Mutation testing relies on the coupling hypothesis: <a href="https://dl.acm.org/doi/abs/10.1145/2635868.2635929" rel="nofollow" target="_blank">mutants are coupled with real bugs</a> if a test suite that is sensitive enough to detect mutants is also sensitive enough to detect the more complex real bugs. Reporting mutants and writing tests to kill them only makes sense if they are coupled with real bugs.</div><div><br /></div><div><br /></div><div><br /></div><div><div>I instinctively thought that fault coupling was real, otherwise I would not have worked on mutation testing at all, and I’ve seen many many cases where mutants pointed to a bug; but still, I wanted to verify this hypothesis, if only for our code base. I designed an experiment: I would generate all mutants in a line for explicit bug-fixing changes, before and after the fix, and I would check whether, had mutation testing been run, it would have surfaced a mutant in the change that introduced the bug, and potentially prevented it (i.e., killed in the change that fixed the bug and added new tests cases). I ran this experiment on weekends for over a month, because we did not have the resources to run it during workdays. While I normally generate a single mutant in a line, to test the fault coupling effect, I used the classical mutation testing approach and generated all possible mutants, while still adhering to the arid node suppression. A total of 33 million test suites were executed to test hundreds of thousands of mutants, finally to conclude that, in around 70% of cases, a bug was coupled with a mutant. </div></div><div><br /></div><div><br /></div><div>While I was at it, I also checked my intuition on whether a single mutant per line was enough, and found that it was overwhelmingly so: in more than 90% of cases, either all mutants were killed in a line or none was. It’s worth keeping in mind that I still applied my arid node suppression heuristics for this experiment. It was great to finally have confirmation of my intuitions.</div><div><br /></div><div><br /></div><div>I also looked into the developer behavior changes after using mutation testing on a project for longer periods of time, and discovered that projects that use mutation testing get more tests over time, as developers get exposed to more and more mutants. Not only do developers write more test cases, but those test cases are more effective in killing mutants: less and less mutants get reported over time. I noticed this from personal experience too: when writing unit tests, I would see where I cut some corners in the tests, and anticipated the mutant. Now I just add the missing test cases, rather than facing a mutant in my Code review, and I rarely see mutants these days, as I’ve learned to anticipate and preempt them.</div><div><br /></div><div><br /></div><div>You can read more about the findings in our <a href="https://homes.cs.washington.edu/~rjust/publ/mutation_testing_practices_icse_2021.pdf">ICSE paper</a>.</div><div><br /></div><div><br /></div><div><h2>Conclusion</h2></div><div><br /></div><div><br /></div><div><div>It’s been a long road since that hackathon in the winter of 2013. Mutation testing was a lot of fun to work on. It had its challenges, and I had days where I thought that I would throw everything down the drain (I’m looking at you, clang), but I am glad I stuck with it.</div></div><div><br /></div><div><br /></div><div><br /></div><div><div>The most interesting part of the project was getting Mutation testing to scale to such a large code base, and that required redefining the problem and adapting it to the existing ecosystem that engineers were already used to. Another interesting angle was working, and learning from, the academic community, in particular Gordon Fraser (University of Passau) and René Just (University of Washington).</div></div><div><br /></div><div><br /></div><div><br /></div><div>I would like to encourage everyone to give one of the many open source mutation testing tools a try on their projects. With some tweaks here and there, it can be a great way to keep your software well tested.</div><div><br /></div><div><br /></div><div><br /></div></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Mutation Testing&url=https://testing.googleblog.com/2021/04/mutation-testing.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2021/04/mutation-testing.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2021/04/mutation-testing.html#comments' style='font-weight: 500; text-decoration: underline;'>3 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2021/04/mutation-testing.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Goran%20Petrovic' rel='tag'> Goran Petrovic </a> , <a class='label' href='https://testing.googleblog.com/search/label/mutation%20testing' rel='tag'> mutation testing </a> </span> </div> </div> </div> <div class='blog-pager' id='blog-pager'> <a class='home-link' href='https://testing.googleblog.com/'> <i class='material-icons'>  </i> </a> <i class='material-icons disabled'>  </i> <span id='blog-pager-older-link'> <a class='blog-pager-older-link' href='https://testing.googleblog.com/search/label/mutation%20testing?updated-max=2021-04-12T08:16:00-07:00&max-results=20&start=20&by-date=false' id='Blog1_blog-pager-older-link' title='Older Posts'> <i class='material-icons'>  </i> </a> </span> </div> <div class='clear'></div> </div></div> </div> </div> <div class='col-right'> <div class='section' id='sidebar-top'><div class='widget HTML' data-version='1' id='HTML8'> <div class='widget-content'> <div class='searchBox'> <input type='text' title='Search This Blog' placeholder='Search blog ...' /> </div> </div> <div class='clear'></div> </div> </div> <div id='aside'> <div class='section' id='sidebar'><div class='widget Label' data-version='1' id='Label1'> <div class='tab'> <img class='sidebar-icon' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII='/> <h2> Labels </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content list-label-widget-content'> <ul> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/TotT'> TotT </a> <span dir='ltr'> 102 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/GTAC'> GTAC </a> <span dir='ltr'> 61 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/James%20Whittaker'> James Whittaker </a> <span dir='ltr'> 42 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Misko%20Hevery'> Misko Hevery </a> <span dir='ltr'> 32 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Code%20Health'> Code Health </a> <span dir='ltr'> 30 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anthony%20Vallone'> Anthony Vallone </a> <span dir='ltr'> 27 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrick%20Copeland'> Patrick Copeland </a> <span dir='ltr'> 23 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jobs'> Jobs </a> <span dir='ltr'> 18 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Andrew%20Trenk'> Andrew Trenk </a> <span dir='ltr'> 12 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/C%2B%2B'> C++ </a> <span dir='ltr'> 11 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund'> Patrik Höglund </a> <span dir='ltr'> 8 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/JavaScript'> JavaScript </a> <span dir='ltr'> 7 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Allen%20Hutchison'> Allen Hutchison </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/George%20Pirocanac'> George Pirocanac </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zhanyong%20Wan'> Zhanyong Wan </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Harry%20Robinson'> Harry Robinson </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Java'> Java </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julian%20Harty'> Julian Harty </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Bender'> Adam Bender </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alberto%20Savoia'> Alberto Savoia </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ben%20Yu'> Ben Yu </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Erik%20Kuefler'> Erik Kuefler </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Philip%20Zembrod'> Philip Zembrod </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shyam%20Seshadri'> Shyam Seshadri </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome'> Chrome </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dillon%20Bly'> Dillon Bly </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Thomas'> John Thomas </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Lesley%20Katzen'> Lesley Katzen </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Kaplan'> Marc Kaplan </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Markus%20Clermont'> Markus Clermont </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander'> Max Kanat-Alexander </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sonal%20Shah'> Sonal Shah </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/APIs'> APIs </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Abhishek%20Arya'> Abhishek Arya </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Myrvold'> Alan Myrvold </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alek%20Icev'> Alek Icev </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Android'> Android </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/April%20Fools'> April Fools </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chaitali%20Narla'> Chaitali Narla </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chris%20Lewis'> Chris Lewis </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome%20OS'> Chrome OS </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Salas'> Diego Salas </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dori%20Reuveni'> Dori Reuveni </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Arbon'> Jason Arbon </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jochen%20Wuttke'> Jochen Wuttke </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kostya%20Serebryany'> Kostya Serebryany </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Eaddy'> Marc Eaddy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87'> Marko Ivanković </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mobile'> Mobile </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Oliver%20Chang'> Oliver Chang </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simon%20Stewart'> Simon Stewart </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stefan%20Kennedy'> Stefan Kennedy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Flakiness'> Test Flakiness </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Titus%20Winters'> Titus Winters </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tony%20Voellm'> Tony Voellm </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/WebRTC'> WebRTC </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yiming%20Sun'> Yiming Sun </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yvette%20Nameth'> Yvette Nameth </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zuri%20Kemp'> Zuri Kemp </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Aaron%20Jacobs'> Aaron Jacobs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Porter'> Adam Porter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Raider'> Adam Raider </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adel%20Saoud'> Adel Saoud </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Faulkner'> Alan Faulkner </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alex%20Eagle'> Alex Eagle </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Amy%20Fu'> Amy Fu </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anantha%20Keesara'> Anantha Keesara </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Antoine%20Picard'> Antoine Picard </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/App%20Engine'> App Engine </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ari%20Shamash'> Ari Shamash </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Arif%20Sukoco'> Arif Sukoco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Benjamin%20Pick'> Benjamin Pick </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bob%20Nystrom'> Bob Nystrom </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bruce%20Leban'> Bruce Leban </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Arguelles'> Carlos Arguelles </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Israel%20Ortiz%20Garc%C3%ADa'> Carlos Israel Ortiz García </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Cathal%20Weakliam'> Cathal Weakliam </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Christopher%20Semturs'> Christopher Semturs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Clay%20Murphy'> Clay Murphy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dagang%20Wei'> Dagang Wei </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Maksimovich'> Dan Maksimovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Shi'> Dan Shi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Willemsen'> Dan Willemsen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Chen'> Dave Chen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Gladfelter'> Dave Gladfelter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Bendory'> David Bendory </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Mandelberg'> David Mandelberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Derek%20Snyder'> Derek Snyder </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Cavalcanti'> Diego Cavalcanti </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov'> Dmitry Vyukov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz'> Eduardo Bravo Ortiz </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ekaterina%20Kamenskaya'> Ekaterina Kamenskaya </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliott%20Karpilovsky'> Elliott Karpilovsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliotte%20Rusty%20Harold'> Elliotte Rusty Harold </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Espresso'> Espresso </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Felipe%20Sodr%C3%A9'> Felipe Sodré </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Francois%20Aube'> Francois Aube </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Gene%20Volovich'> Gene Volovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Google%2B'> Google+ </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goran%20Petrovic'> Goran Petrovic </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goranka%20Bjedov'> Goranka Bjedov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hank%20Duan'> Hank Duan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Havard%20Rast%20Blok'> Havard Rast Blok </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hongfei%20Ding'> Hongfei Ding </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Elbaum'> Jason Elbaum </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Huggins'> Jason Huggins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jay%20Han'> Jay Han </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Hoy'> Jeff Hoy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Listfield'> Jeff Listfield </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jessica%20Tomechak'> Jessica Tomechak </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jim%20Reardon'> Jim Reardon </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joe%20Allan%20Muharsky'> Joe Allan Muharsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joel%20Hynoski'> Joel Hynoski </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Micco'> John Micco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Penix'> John Penix </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway'> Jonathan Rockway </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez'> Jonathan Velasquez </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Josh%20Armour'> Josh Armour </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julie%20Ralph'> Julie Ralph </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kai%20Kent'> Kai Kent </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Karin%20Lundberg'> Karin Lundberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kaue%20Silveira'> Kaue Silveira </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Bourrillion'> Kevin Bourrillion </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Graney'> Kevin Graney </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kirkland'> Kirkland </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever'> Kurt Alfred Kluever </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Manjusha%20Parvathaneni'> Manjusha Parvathaneni </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis'> Marek Kiszkis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marius%20Latinis'> Marius Latinis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Ivey'> Mark Ivey </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Manley'> Mark Manley </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Striebeck'> Mark Striebeck </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Matt%20Lowrie'> Matt Lowrie </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Meredith%20Whittaker'> Meredith Whittaker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Bachman'> Michael Bachman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Klepikov'> Michael Klepikov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Aizatsky'> Mike Aizatsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Wacker'> Mike Wacker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy'> Mona El Mahdy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Noel%20Yap'> Noel Yap </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Palak%20Bansal'> Palak Bansal </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patricia%20Legaspi'> Patricia Legaspi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Per%20Jacobsson'> Per Jacobsson </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht'> Peter Arrenbrecht </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Spragins'> Peter Spragins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Norman'> Phil Norman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Rollet'> Phil Rollet </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Pooja%20Gupta'> Pooja Gupta </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Project%20Showcase'> Project Showcase </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Radoslav%20Vasilev'> Radoslav Vasilev </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Dewan'> Rajat Dewan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Jain'> Rajat Jain </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rich%20Martin'> Rich Martin </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Richard%20Bustamante'> Richard Bustamante </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Roshan%20Sembacuttiaratchy'> Roshan Sembacuttiaratchy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov'> Ruslan Khamitov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sam%20Lee'> Sam Lee </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sean%20Jordan'> Sean Jordan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sharon%20Zhou'> Sharon Zhou </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shiva%20Garg'> Shiva Garg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Siddartha%20Janga'> Siddartha Janga </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simran%20Basi'> Simran Basi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stan%20Chan'> Stan Chan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stephen%20Ng'> Stephen Ng </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tejas%20Shah'> Tejas Shah </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Analytics'> Test Analytics </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Engineer'> Test Engineer </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tim%20Lyakhovetskiy'> Tim Lyakhovetskiy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tom%20O%27Neill'> Tom O'Neill </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Vojta%20J%C3%ADna'> Vojta Jína </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/automation'> automation </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/dead%20code'> dead code </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/iOS'> iOS </a> <span dir='ltr'> 1 </span> </li> <li> <span dir='ltr'> mutation testing </span> <span dir='ltr'> 1 </span> </li> </ul> <div class='clear'></div> </div> </div><div class='widget BlogArchive' data-version='1' id='BlogArchive1'> <div class='tab'> <i class='material-icons icon'>  </i> <h2> Archive </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content'> <div id='ArchiveList'> <div id='BlogArchive1_ArchiveList'> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/'> 2024 </a> <span class='post-count' dir='ltr'>(12)</span> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2024/10/smurf-beyond-test-pyramid.html'> SMURF: Beyond the Test Pyramid </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/'> 2023 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/09/'> Sep </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/'> 2022 </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/'> 2021 </a> <span class='post-count' dir='ltr'>(3)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/'> 2020 </a> <span class='post-count' dir='ltr'>(8)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/'> 2019 </a> <span class='post-count' dir='ltr'>(4)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/'> 2018 </a> <span class='post-count' dir='ltr'>(7)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/'> 2017 </a> <span class='post-count' dir='ltr'>(17)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/'> 2016 </a> <span class='post-count' dir='ltr'>(15)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/'> 2015 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/'> 2014 </a> <span class='post-count' dir='ltr'>(24)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/06/'> Jun </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/'> 2013 </a> <span class='post-count' dir='ltr'>(16)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/'> 2012 </a> <span class='post-count' dir='ltr'>(11)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/'> 2011 </a> <span class='post-count' dir='ltr'>(39)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/06/'> Jun </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/05/'> May </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/01/'> Jan </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/'> 2010 </a> <span class='post-count' dir='ltr'>(37)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/11/'> Nov </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/10/'> Oct </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/03/'> Mar </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/'> 2009 </a> <span class='post-count' dir='ltr'>(54)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/07/'> Jul </a> <span class='post-count' dir='ltr'>(15)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/06/'> Jun </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/'> 2008 </a> <span class='post-count' dir='ltr'>(75)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/12/'> Dec </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/11/'> Nov </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/10/'> Oct </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/08/'> Aug </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/07/'> Jul </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/06/'> Jun </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/05/'> May </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/04/'> Apr </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/02/'> Feb </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/'> 2007 </a> <span class='post-count' dir='ltr'>(41)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/10/'> Oct </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/04/'> Apr </a> <span class='post-count' dir='ltr'>(7)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/03/'> Mar </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> </div> </div> <div class='clear'></div> </div> </div><div class='widget HTML' data-version='1' id='HTML6'> <div class='widget-content'> <a href="http://googletesting.blogspot.com/atom.xml"> <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==" class="sidebar-icon" /> <h2>Feed</h2> </a> </div> <div class='clear'></div> </div></div> <div class='section' id='sidebar-bottom'><div class='widget HTML' data-version='1' id='HTML9'> <div class='widget-content'> <a href='http://cloud.feedly.com/#subscription%2Ffeed%2Fhttp%3A%2F%2Fgoogletesting.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='blank'><img id="feedlyFollow" src="https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_t1pNpywZOzPfRrQOF5Y-ebEmI5I3hneDw-i2iTxXtyuo2ps2NUA78WNe0qcq3FJPL6BQlt6JScFGPaMvrrAB_Vs2WUAtnycyEdWA_68jHxfr7dERmcX446__M2rWcujB90AMZgYyB3iJsvZ1WE7Dw=s0-d" alt="follow us in feedly" width="66" height="20"></a> <div class="share followgooglewrapper"> <button data-href="https://twitter.com/intent/follow?original_referer=http://googletesting.blogspot.com/&screen_name=googletesting" onclick='sharingPopup(this);' id='twitter-share'><span class="twitter-follow">Follow @googletesting</span></button> <script> function sharingPopup (button) { var url = button.getAttribute("data-href"); window.open( url,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); } </script> </div> </div> <div class='clear'></div> </div></div> </div> </div> <div style='clear:both;'></div> </div>  <div class='google-footer-outer loading'> <div id='google-footer'> <a href='//www.google.com/'> <img class='google-logo-dark' height='36' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==' style='margin-top: -16px;' width='92'/> </a> <ul> <li> <a href='//www.google.com/'> Google </a> </li> <li> <a href='//www.google.com/policies/privacy/'> Privacy </a> </li> <li> <a href='//www.google.com/policies/terms/'> Terms </a> </li> </ul> </div> </div> <script type='text/javascript'> //<![CDATA[ // Social sharing popups. var postEl = document.getElementsByClassName('social-wrapper'); var postCount = postEl.length; for(i=0; i<postCount;i++){ postEl[i].addEventListener("click", function(event){ var postUrl = this.getAttribute("data-href"); window.open( postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); });} //]]> </script> <script type='text/javascript'> //<![CDATA[ var BreakpointHandler = function() { this.initted = false; this.isHomePage = false; this.isMobile = false; }; BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) { // Use $.trim for IE8 compatibility summaryHtml = $.trim(summaryHtml).replace(/(<br>|\s)+$/,''); if (lastNode.nodeType == 3) { var lastChar = summaryHtml.slice(-1); if (!lastChar.match(/[.”"?]/)) { if (!lastChar.match(/[A-Za-z]/)) { summaryHtml = summaryHtml.slice(0, -1); } summaryHtml += ' ...'; } } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) { summaryHtml += ' ...'; } return summaryHtml; }; BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) { var seenWords = 0; var summaryHtml = ''; for (var i=0; i < content.childNodes.length; i++) { var node = content.childNodes[i]; var nodeText; if (node.nodeType == 1) { if (node.hasAttribute('data-about-pullquote')) { continue; } nodeText = node.textContent; if (nodeText === undefined) { // innerText for IE8 nodeText = node.innerText; } if (node.nodeName == 'DIV' || node.nodeName == 'B') { // Don't end early if we haven't seen enough words. if (seenWords < 10) { continue; } if (i > 0) { summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]); } break; } summaryHtml += node.outerHTML; } else if (node.nodeType == 3) { nodeText = node.nodeValue; summaryHtml += nodeText + ' '; } var words = nodeText.match(/\S+\s*/g); if (!words) { continue; } var remain = numWords - seenWords; if (words.length >= remain) { summaryHtml = this.finalizeSummary(summaryHtml, node); break; } seenWords += words.length; } return summaryHtml; }; BreakpointHandler.prototype.detect = function() { var match, pl = /\+/g, search = /([^&=]+)=?([^&]*)/g, decode = function (s) { return decodeURIComponent(s.replace(pl, " ")); }, query = window.location.search.substring(1); var urlParams = {}; while (match = search.exec(query)) urlParams[decode(match[1])] = decode(match[2]); this.isListPage = $('html').hasClass('list-page'); this.isMobile = urlParams['m'] === '1'; this.isHomePage = window.location.pathname == '/'; }; BreakpointHandler.prototype.initContent = function() { var self = this; $('.post').each(function(index) { var body = $(this).children('.post-body')[0]; var content = $(body).children('.post-content')[0]; $(content).addClass('post-original'); var data = $(content).children('script').html(); data = self.rewriteForSSL(data); // If exists, extract specified editor's preview. var match = data.match(/([\s\S]+?)<div data-is-preview.+?>([\s\S]+)<\/div>/m); if (match) { data = match[1]; } // Prevent big images from loading when they aren't needed. // This must be done as a pre-injection step, since image loading can't be // canceled once embedded into the DOM. if (self.isListPage && self.isMobile) { data = data.replace(/<(img|iframe) .+?>/g, ''); } // Insert template to be rendered as nodes. content.innerHTML = data; if (self.isListPage) { var summary = document.createElement('div'); $(summary).addClass('post-content'); $(summary).addClass('post-summary'); body.insertBefore(summary, content); if (match) { // Use provided summary. summary.innerHTML = match[2]; } else { // Generate a summary. // Summary generation relies on DOM, so it must occur after content is // inserted into the page. summary.innerHTML = self.generateSummaryFromContent(content, 30); } // Add read more link to summary. var titleAnchor = $(this).find('.title a')[0]; var link = titleAnchor.cloneNode(true); link.innerHTML = 'Read More'; $(link).addClass('read-more'); summary.appendChild(link); } }); // Firefox does not allow for proper styling of BR. if (navigator.userAgent.indexOf('Firefox') > -1) { $('.post-content br').replaceWith('<span class="space"></span>'); } $('.loading').removeClass('loading'); }; BreakpointHandler.prototype.process = function() { if (!this.initted) { var makeInsecureImageRegex = function(hosts) { var whitelist = hosts.join('|').replace(/\./g,'\\.'); // Normal image tags, plus input images (yes, this is possible!) return new RegExp('(<(img|input)[^>]+?src=("|\'))http:\/\/(' + whitelist +')', 'g'); }; this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS); this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]); this.detect(); this.initContent(); this.initted = true; } }; BreakpointHandler.KNOWN_HTTPS_HOSTS = [ "www.google.org", "www.google.com", "services.google.com", "blogger.com", "draft.blogger.com", "www.blogger.com", "photos1.blogger.com", "photos2.blogger.com", "photos3.blogger.com", "blogblog.com", "img1.blogblog.com", "img2.blogblog.com", "www.blogblog.com", "www1.blogblog.com", "www2.blogblog.com", "0.bp.blogspot.com", "1.bp.blogspot.com", "2.bp.blogspot.com", "3.bp.blogspot.com", "4.bp.blogspot.com", "lh3.googleusercontent.com", "lh4.googleusercontent.com", "lh5.googleusercontent.com", "lh6.googleusercontent.com", "themes.googleusercontent.com", ]; BreakpointHandler.prototype.rewriteForSSL = function(html) { // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content. return html.replace(this.sslImageRegex, '$1https://$4') .replace(this.sslImageCurrentDomainRegex, '$1//$4') .replace(/(<(embed|iframe)[^>]+?src=("|'))http:\/\/([^"']*?(youtube|picasaweb\.google)\.com)/g, '$1https://$4') // Slideshow SWF takes a image host, so we need to rewrite that parameter. .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s'); }; $(document).ready(function() { var handler = new BreakpointHandler(); handler.process(); // Top-level navigation. $(".BlogArchive .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); $(".Label .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); // Blog archive year expansion. $('.BlogArchive .intervalToggle').click(function(ev) { ev.preventDefault(); if ($(this).parent().hasClass('collapsed')) { $(this).parent().removeClass('collapsed'); $(this).parent().addClass('expanded'); } else { $(this).parent().removeClass('expanded'); $(this).parent().addClass('collapsed'); } }); // Reverse order of months. $('.BlogArchive .intervalToggle + div').each(function(_, items) { var year = $(this); year.children().each(function(_, month) { year.prepend(month); }); }); // Set anchors to open in new tab. $('.post-content img').parent().each(function(_, node) { if (node.nodeName == 'A') { $(this).attr('target', '_blank'); } }); // Process search requests. $('.searchBox input').on("keypress", function(ev) { if (ev.which == 13) { window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val()); } }); }); (function($, window) { var archiveButton = $($('#sidebar .widget.BlogArchive h2')[0]); var folderIcon = $('#sidebar .widget.BlogArchive h2::after'); var archivePanel = $('#BlogArchive1'); archiveButton.click(function(e) { if (archivePanel.hasClass('archive-open')) { // It's open, so now we close it archivePanel.removeClass('archive-open'); archivePanel.addClass('archive-closed'); archiveButton.removeClass('archive-open'); archiveButton.addClass('archive-closed'); archivePanel.css('height', '60px'); } else { // It's closed, so open it archivePanel.removeClass('archive-closed'); archivePanel.addClass('archive-open'); archiveButton.removeClass('archive-closed'); archiveButton.addClass('archive-open'); archivePanel.css('height', 'auto'); folderIcon.css('content', 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEggkDzi8QVSua_XxLOnfxWE_8nQc1MAhRPBxJej91JW2ZBbKvb2TjaNf66ZoTqCrBivwRr-TLJYxFm2ZAiMd-zfbJ0TjblXGvUszWFbdRncNOL7jseiJCL9uQoqN8BM8FpesdSw/s1600/keyboard_arrow_up_grey600_24dp.png'); } }); })($, window); (function ($, window, devsite) { devsite.devsite = {}; devsite.expandLeftNav = devsite.expandLeftNav || {}; devsite.expandLeftNav.ANALYTICS_LABEL_ = 'Hamburger menu'; devsite.expandLeftNav.ANALYTICS_ACTION_OPEN_ = 'Open'; devsite.expandLeftNav.ANALYTICS_ACTION_CLOSE_ = 'Close'; devsite.expandLeftNav.lastState_ = 0; devsite.expandLeftNav.chekovEnabled_ = false; devsite.expandLeftNav.init = function () { devsite.expandLeftNav.chekovEnabled_ = true; var navElement; if (devsite.expandLeftNav.chekovEnabled_) { navElement = $('.devsite-nav-responsive'); } else { navElement = $('.devsite-section-nav-responsive'); } this.drawerWidth = parseInt(navElement.css('width')); navElement.css({ 'left': -(this.drawerWidth) }); var expandButton = $('.devsite-expand-section-nav'); expandButton.click(function () { devsite.expandLeftNav.handleNavigationOpened(); }); navElement.find('.devsite-nav-responsive-forward').click( devsite.expandLeftNav.openPanel); navElement.find('.devsite-nav-responsive-back').click( devsite.expandLeftNav.closePanel); if ($('.devsite-nav-responsive-tabs-panel + ' + '.devsite-nav-responsive-sidebar-panel').length) { devsite.expandLeftNav.openPanel(); } }; devsite.expandLeftNav.handleNavigationOpened = function (opt_noAnimate) { var mask; if (devsite.expandLeftNav.chekovEnabled_) { var nav = $('.devsite-nav-responsive'); if (opt_noAnimate) { nav.addClass('devsite-nav-responsive-no-animate'); mask = devsite.devsite.showSiteMask(0); } else { mask = devsite.devsite.showSiteMask(); } nav.addClass('devsite-nav-responsive-open'); } else { devsite.expandLeftNav.lastState_ = devsite.sticky.currentState; devsite.sticky.setState(devsite.sticky.state.COLLAPSED_HEADER); var headerHeight = devsite.sticky.getHeaderHeight() - devsite.sticky.desiredMargin; var drawerHeight = $(window).height() - headerHeight; mask = devsite.devsite.showArticleMask(); $('.devsite-section-nav-responsive').css({ 'height': drawerHeight, 'left': '0', 'top': headerHeight, 'visibility': 'visible' }).focus(); } $(mask).click(devsite.expandLeftNav.handleNavigationClosed); $(document).on('keydown.escape', function (e) { if (e.keyCode == $.ui.keyCode.ESCAPE) { devsite.expandLeftNav.handleNavigationClosed(); $(document).off('keydown.escape'); } }); }; devsite.expandLeftNav.getScrollTop = function () { return $(window).scrollTop(); }; devsite.expandLeftNav.handleNavigationClosed = function () { var nav; if (devsite.expandLeftNav.chekovEnabled_) { nav = $('.devsite-nav-responsive'); devsite.devsite.hideSiteMask(); nav.removeClass('devsite-nav-responsive-open ' + 'devsite-nav-responsive-no-animate'); } else { nav = $('.devsite-section-nav-responsive'); devsite.devsite.hideArticleMask(); devsite.sticky.setState(devsite.expandLeftNav.lastState_); nav .css('left', -(devsite.expandLeftNav.drawerWidth)) .one('transitionend', function () { nav.css('visibility', 'hidden'); }); } nav.scrollTop(0); }; devsite.expandLeftNav.openPanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel.show(); parentPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel .removeClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); parentPanel.hide(); }); setTimeout(function () { childPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); }, 1); }; devsite.expandLeftNav.closePanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); parentPanel .show() .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); setTimeout(function () { parentPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); childPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel.hide(); parentPanel.removeClass('devsite-nav-responsive-transition'); }); }, 1); }; devsite.expandLeftNav.FADE_SLOW_ = 'slow'; devsite.expandLeftNav.FADE_FAST_ = 'fast'; devsite.expandLeftNav.SITE_MASK_CSS_ = '.devsite-site-mask'; devsite.devsite.showSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_SLOW_; } devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, false, opt_animate); }; devsite.devsite.hideSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_FAST_; } devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, true, opt_animate); }; devsite.devsite.showArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_('.devsite-article-mask', false, devsite.expandLeftNav.FADE_SLOW_); }; devsite.devsite.hideArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_('.devsite-article-mask', true, devsite.expandLeftNav.FADE_FAST_); }; devsite.devsite.setMask_ = function(className, out, opt_fadeTime) { var query = $(className); if (opt_fadeTime === 0) { out ? query.hide() : query.show(); } else { out ? query.fadeOut(opt_fadeTime) : query.fadeIn(opt_fadeTime); } return $(className)[0]; }; devsite.devsite.setMouseScrollingEnabled = function(trueOrFalse) { if (trueOrFalse == true) { $('html, body').css({ 'overflow': '' }); } else { $('html, body').css({ 'overflow': 'hidden' }); } }; })($, window, devsite = {}); if (window.jQuery) { $(document).ready(function () { if (window.devsite) { devsite.expandLeftNav.init(); } }); } //]]> </script> <style> .widget ul{ line-height: 1.6 !important; } #sidebar ul li a{ color:black; line-height: 20px; } #sidebar ul li a:hover{ color:#4184F3; } </style> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY6DXzXxIm8zacSFlkEqTOUMt_CRPg:1732452429087';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d15045980','//testing.googleblog.com/search/label/mutation%20testing','15045980'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '15045980', 'title': 'Google Testing Blog', 'url': 'https://testing.googleblog.com/search/label/mutation%20testing', 'canonicalUrl': 'https://testing.googleblog.com/search/label/mutation%20testing', 'homepageUrl': 'https://testing.googleblog.com/', 'searchUrl': 'https://testing.googleblog.com/search', 'canonicalHomepageUrl': 'https://testing.googleblog.com/', 'blogspotFaviconUrl': 'https://testing.googleblog.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'G-838ZCPQWM6', 'analytics4': true, 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Google Testing Blog - RSS\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/15045980/posts/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/d78375fb222d99b3', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'index', 'searchLabel': 'mutation testing', 'pageName': 'mutation testing', 'pageTitle': 'Google Testing Blog: mutation testing'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Google Testing Blog', 'description': '', 'url': 'https://testing.googleblog.com/search/label/mutation%20testing', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': false, 'isSearch': true, 'isLabelSearch': true, 'search': {'label': 'mutation testing', 'resultsMessage': 'Showing posts with the label mutation testing', 'resultsMessageHtml': 'Showing posts with the label \x3cspan class\x3d\x27search-label\x27\x3emutation testing\x3c/span\x3e'}}}]); _WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'navMessage': 'Showing posts with label \x3cb\x3emutation testing\x3c/b\x3e. \x3ca href\x3d\x22https://testing.googleblog.com/\x22\x3eShow all posts\x3c/a\x3e', 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/2646514562-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML9', 'sidebar-bottom', document.getElementById('HTML9'), {}, 'displayModeFull')); </script> </body> </html>