CINXE.COM

<!DOCTYPE html> <html class='v2 list-page' dir='ltr' itemscope='' itemtype='http://schema.org/Blog' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <title> Google Testing Blog: Test Flakiness </title> <meta content='width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0' name='viewport'/> <meta content='IE=Edge' http-equiv='X-UA-Compatible'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='en_US' property='og:locale'/> <meta content='https://testing.googleblog.com/search/label/Test%20Flakiness' property='og:url'/> <meta content='Google Testing Blog' property='og:site_name'/>  <meta content='Google Testing Blog' property='og:title'/> <meta content='summary' name='twitter:card'/> <meta content='@googletesting' name='twitter:creator'/> <link href='https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic' rel='stylesheet' type='text/css'/> <link href='https://fonts.googleapis.com/icon?family=Material+Icons' rel='stylesheet'/> <script src='https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js' type='text/javascript'></script>  <style id='page-skin-1' type='text/css'></style> <style id='template-skin-1' type='text/css'></style>  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://testing.googleblog.com/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://testing.googleblog.com/search/label/Test%20Flakiness' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://testing.googleblog.com/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="Google Testing Blog - RSS" href="https://testing.googleblog.com/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://www.blogger.com/feeds/15045980/posts/default" />  <meta content='https://testing.googleblog.com/search/label/Test%20Flakiness' property='og:url'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='' property='og:description'/>  <base target='_self'/> <style> html { font-family: Roboto, sans-serif; -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; } body { padding: 0; /* This ensures that the scroll bar is always present, which is needed */ /* because content render happens after page load; otherwise the header */ /* would "bounce" in-between states. */ min-height: 150%; } h2 { font-size: 20px; } h1, h2, h3, h4, h5 { line-height: 2em; } html, h4, h5, h6 { font-size: 17px; } h3 { font-size: 18px !important; } a, a:visited { color: #4184F3; text-decoration: none; } a:focus, a:hover, a:active { text-decoration: none; } .Header { margin-top: 15px; } .Header h1 { font-size: 32px; font-weight: 300; line-height: 32px; height: 42px; } .header-inner .Header .titlewrapper { padding: 0; margin-top: 30px; } .header-inner .Header .descriptionwrapper { padding: 0; margin: 0; } .cols-wrapper { margin-top: 56px; } .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 60px; } .header-inner { height: 256px; position: relative; } html, .header-inner a { color: #212121; color: rgba(0,0,0,.87); } .header-inner .google-logo { display: inline-block; background-size: contain; z-index: 1; height: 46px; overflow: hidden; margin-top: 4px; margin-right: 8px; } .header-left { position: absolute; top: 50%; -webkit-transform: translateY(-50%); transform: translateY(-50%); margin-top: 12px; width: 100%; } .google-logo { margin-left: -4px; } #google-footer { position: relative; font-size: 13px; list-style: none; text-align: right; } #google-footer a { color: #444; } #google-footer ul { margin: 0; padding: 0; height: 144px; line-height: 144px; } #google-footer ul li { display: inline; } #google-footer ul li:before { color: #999; content: "\00b7"; font-weight: bold; margin: 5px; } #google-footer ul li:first-child:before { content: ''; } #google-footer .google-logo-dark { left: 0; margin-top: -16px; position: absolute; top: 50%; } /** Sitemap links. **/ .footer-inner-2 { font-size: 14px; padding-top: 42px; padding-bottom: 74px; } .footer-inner-2 .HTML h2 { color: #212121; color: rgba(0,0,0,.87); font-size: 14px; font-weight: 500; padding-left: 0; margin: 10px 0; } .footer-inner-2 .HTML ul { font-weight: normal; list-style: none; padding-left: 0; } .footer-inner-2 .HTML li { line-height: 24px; padding: 0; } .footer-inner-2 li a { color: rgba(65,132,243,.87); } /** Archive widget. **/ .BlogArchive { font-size: 13px; font-weight: normal; } .BlogArchive .widget-content { display: none; } .BlogArchive h2, .Label h2 { color: #4184F3; text-decoration: none; } .BlogArchive .hierarchy li { display: inline-block; } /* Specificity needed here to override widget CSS defaults. */ .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li { padding-left: 0; text-indent: 0; } .BlogArchive .intervalToggle { cursor: pointer; } .BlogArchive .expanded .intervalToggle .new-toggle { -ms-transform: rotate(180deg); transform: rotate(180deg); } .BlogArchive .new-toggle { float: right; padding-top: 3px; opacity: 0.87; } #ArchiveList { text-transform: uppercase; } #ArchiveList .expanded > ul:last-child { margin-bottom: 16px; } #ArchiveList .archivedate { width: 100%; } /* Months */ .BlogArchive .items { max-width: 150px; margin-left: -4px; } .BlogArchive .expanded .items { margin-bottom: 10px; overflow: hidden; } .BlogArchive .items > ul { float: left; height: 32px; } .BlogArchive .items a { padding: 0 4px; } .Label { font-size: 13px; font-weight: normal; } .sidebar-icon { display: inline-block; width: 24px; height: 24px; vertical-align: middle; margin-right: 12px; margin-top: -1px } .Label a { margin-right: 4px; } .Label .widget-content { display: none; } .FollowByEmail { font-size: 13px; font-weight: normal; } .FollowByEmail h2 { background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); background-repeat: no-repeat; background-position: 0 50%; text-indent: 30px; } .FollowByEmail .widget-content { display: none; } .searchBox input { border: 1px solid #eee; color: #212121; color: rgba(0,0,0,.87); font-size: 14px; padding: 8px 8px 8px 40px; width: 164px; font-family: Roboto, sans-serif; background: url("https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png") 8px center no-repeat; } .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */ color: rgba(0,0,0,.54); } .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */ color: #000; opacity: 0.54; } .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */ color: #000; opacity: 0.54; } .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */ color: #757575; } .widget-item-control { margin-top: 0px; } .section { margin: 0; padding: 0; } #sidebar-top { border: 1px solid #eee; } #sidebar-top > div { margin: 16px 0; } .widget ul { line-height: 1.6; } /*main post*/ .post { margin-bottom:30px; } #main .post .title { margin: 0; } #main .post .title a { color: #212121; color: rgba(0,0,0,.87); font-weight: normal; font-size: 24px; } #main .post .title a:hover { text-decoration:none; color:#4184F3; } .message, #main .post .post-header { margin: 0; padding: 0; } #main .post .post-header .caption, #main .post .post-header .labels-caption, #main .post .post-footer .caption, #main .post .post-footer .labels-caption { color: #444; font-weight: 500; } #main .tr-caption-container td { text-align: left; } #main .post .tr-caption { color: #757575; color: rgba(0,0,0,.54); display: block; max-width: 560px; padding-bottom: 20px; } #main .post .tr-caption-container { line-height: 24px; margin: -1px 0 0 0 !important; padding: 4px 0; text-align: left; } #main .post .post-header .published{ font-size:11px; font-weight:bold; } .post-header .publishdate { font-size: 17px; font-weight:normal; color: #757575; color: rgba(0,0,0,.54); } #main .post .post-footer{ font-size:12px; padding-bottom: 21px; } .label-footer { margin-bottom: 12px; margin-top: 12px; } .comment-img { margin-right: 16px; opacity: 0.54; vertical-align: middle; } #main .post .post-header .published { margin-bottom: 40px; margin-top: -2px; } .post .post-content { color: #212121; color: rgba(0,0,0,.87); font-size: 17px; margin: 25px 0 36px 0; line-height: 32px; word-wrap: break-word; } .post-body .post-content ul, .post-body .post-content ol { margin: 16px 0; padding: 0 48px; } .post-summary { display: none; } /* Capture section headers. */ .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b { display: inline-block; margin-bottom: 8px; margin-top: 24px; } .post-content li { line-height: 32px; } /* Override all post images/videos to left align. */ .post-content .separator, .post-content > div { text-align: left; } .post-content .separator > a, .post-content .separator > span { margin-left: 0 !important; } .post-content img { max-width: 100%; } .post-content .tr-caption-container img { margin-bottom: 12px; } .post-content iframe, .post-content embed { max-width: 100%; } .post-content .carousel-container { margin-bottom: 48px; } #main .post-content b { font-weight: 500; } /* These are the main paragraph spacing tweaks. */ #main .post-content br { content: ' '; display: block; padding: 4px; } .post-content .space { display: block; height: 8px; } .post-content iframe + .space, .post-content iframe + br { padding: 0 !important; } #main .post .jump-link { margin-bottom:10px; } .post-content img, .post-content iframe { margin: 30px 0 20px 0; } .post-content > img:first-child, .post-content > iframe:first-child { margin-top: 0; } .col-right .section { padding: 0 16px; } #aside { background:#fff; border:1px solid #eee; border-top: 0; } #aside .widget { margin:0; } #aside .widget h2, #ArchiveList .toggle + a.post-count-link { color: #212121; color: rgba(0,0,0,.87); font-weight: 400 !important; margin: 0; } #ArchiveList .toggle { float: right; } #ArchiveList .toggle .material-icons { padding-top: 4px; } #sidebar .tab { cursor: pointer; } #sidebar .tab .arrow { display: inline-block; float: right; } #sidebar .tab .icon { display: inline-block; vertical-align: top; height: 24px; width: 24px; margin-right: 13px; margin-left: -1px; margin-top: 1px; color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget-content > :first-child { padding-top: 8px; } #sidebar .active .tab .arrow { -ms-transform: rotate(180deg); transform: rotate(180deg); } #sidebar .arrow { color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget h2 { font-size: 14px; line-height: 24px; display: inline-block; } #sidebar .widget .BlogArchive { padding-bottom: 8px; } #sidebar .widget { border-bottom: 1px solid #eee; box-shadow: 0px 1px 0 white; margin-bottom: 0; padding: 14px 0; min-height: 20px; } #sidebar .widget:last-child { border-bottom: none; box-shadow: none; margin-bottom: 0; } #sidebar ul { margin: 0; padding: 0; } #sidebar ul li { list-style:none; padding:0; } #sidebar ul li a { line-height: 32px; } #sidebar .archive { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC"); height: 24px; line-height: 24px; padding-left: 30px; } #sidebar .labels { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC"); height: 20px; line-height: 20px; padding-left: 30px; } #sidebar .rss a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII="); } #sidebar .subscription a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); } #sidebar-bottom { background: #f5f5f5; border-top:1px solid #eee; } #sidebar-bottom .widget { border-bottom: 1px solid #e0e0e0; padding: 15px 0; text-align: center; } #sidebar-bottom > div:last-child { border-bottom: 0; } #sidebar-bottom .text { line-height: 20px; } /* Home, forward, and backward pagination. */ .blog-pager { border-top : 1px #e0e0e0 solid; padding-top: 10px; margin-top: 15px; text-align: right !important; } #blog-pager { margin-botom: 0; margin-top: -14px; padding: 16px 0 0 0; } #blog-pager a { display: inline-block; } .blog-pager i.disabled { opacity: 0.2 !important; } .blog-pager i { color: black; margin-left: 16px; opacity: 0.54; } .blog-pager i:hover, .blog-pager i:active { opacity: 0.87; } #blog-pager-older-link, #blog-pager-newer-link { float: none; } .gplus-profile { background-color: #fafafa; border: 1px solid #eee; overflow: hidden; width: 212px; } .gplus-profile-inner { margin-left: -1px; margin-top: -1px; } /* Sidebar follow buttons. */ .followgooglewrapper { padding: 12px 0 0 0; } .loading { visibility: hidden; } .detail-page .post-footer .cmt_iframe_holder { padding-top: 40px !important; } /** Desktop **/ @media (max-width: 900px) { .col-right { display: none; } .col-main { margin-right: 0; min-width: initial; } .footer-outer { display: none; } .cols-wrapper { min-width: initial; } .google-footer-outer { background-color: #f5f5f5; } } /** Tablet **/ @media (max-width: 712px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 40px; } } /* An extra breakpoint accommodating for long blog titles. */ @media (max-width: 600px) { .header-left { height: 100%; position: initial; top: inherit; margin-top: 0; -webkit-transform: initial; transform: initial; } .header-title { margin-top: 18px; } .header-inner { height: auto; margin-bottom: 32px; margin-top: 32px; } .header-desc { margin-top: 12px; } .header-inner .google-logo { height: 40px; margin-top: 3px; } .header-inner .google-logo img { height: 42px; } .header-title h2 { font-size: 32px; line-height: 40px; } } /** Mobile/small desktop window; also landscape. **/ @media (max-width: 480px), (max-height: 480px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 16px; } .cols-wrapper { margin-top: 0; } .post-header .publishdate, .post .post-content { font-size: 16px; } .post .post-content { line-height: 28px; margin-bottom: 30px; } .post { margin-top: 30px; } .byline-author { display: block; font-size: 12px; line-height: 24px; margin-top: 6px; } #main .post .title a { font-weight: 500; color: #4c4c4c; color: rgba(0,0,0,.70); } #main .post .post-header { padding-bottom: 12px; } #main .post .post-header .published { margin-bottom: -8px; margin-top: 3px; } .post .read-more { display: block; margin-top: 14px; } .post .tr-caption { font-size: 12px; } #main .post .title a { font-size: 20px; line-height: 30px; } .post-content iframe { /* iframe won't keep aspect ratio when scaled down. */ max-height: 240px; } .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe { margin-left: -16px; max-width: inherit; width: calc(100% + 32px); } .post-content table, .post-content td { width: 100%; } #blog-pager { margin: 0; padding: 16px 0; } /** List page tweaks. **/ .list-page .post-original { display: none; } .list-page .post-summary { display: block; } .list-page .comment-container { display: none; } .list-page #blog-pager { padding-top: 0; border: 0; margin-top: -8px; } .list-page .label-footer { display: none; } .list-page #main .post .post-footer { border-bottom: 1px solid #eee; margin: -16px 0 0 0; padding: 0 0 20px 0; } .list-page .post .share { display: none; } /** Detail page tweaks. **/ .detail-page .post-footer .cmt_iframe_holder { padding-top: 32px !important; } .detail-page .label-footer { margin-bottom: 0; } .detail-page #main .post .post-footer { padding-bottom: 0; } .detail-page #comments { display: none; } } [data-about-pullquote], [data-is-preview], [data-about-syndication] { display: none; } </style> <noscript> <style> .loading { visibility: visible }</style> </noscript>  <script async='true' src='https://www.googletagmanager.com/gtag/js?id=G-838ZCPQWM6'></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-838ZCPQWM6'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <script type='text/javascript'> //<![CDATA[ var axel = Math.random() + ""; var a = axel * 10000000000000; document.write('<iframe src="https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?" width="1" height="1" frameborder="0" style="display:none"></iframe>'); //]]> </script> <noscript> <img alt='' height='1' src='https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?' width='1'/> </noscript>  <div class='header-outer'> <div class='header-inner'> <div class='section' id='header'><div class='widget Header' data-version='1' id='Header1'> <div class='header-left'> <div class='header-title'> <a class='google-logo' href='https://testing.googleblog.com/'> <img height='50' src='https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png'/> </a> <a href='/.'> <h2> Testing Blog </h2> </a> </div> <div class='header-desc'> </div> </div> </div></div> </div> </div>  <div class='cols-wrapper loading'> <div class='col-main-wrapper'> <div class='col-main'> <div class='section' id='main'><div class='widget Blog' data-version='1' id='Blog1'> <div class='post' data-id='8859994293759789916' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2021/03/test-flakiness-one-of-main-challenges.html' itemprop='url' title='Test Flakiness - One of the main challenges of automated testing (Part II)'> Test Flakiness - One of the main challenges of automated testing (Part II) </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Wednesday, March 24, 2021 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By George Pirocanac<div><br /></div><div>This is part two of a series on test flakiness. The <a href="https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html">first article</a> discussed the four components under which tests are run and the possible reasons for test flakiness. This article will discuss the triage tips and remedies for flakiness for each of these possible reasons.</div><h3 style="text-align: left;"><br /></h3><h3 style="text-align: left;">Components</h3><div><br /></div><div>To review, the four components where flakiness can occur include:</div><div><ul style="text-align: left;"><li>The tests themselves</li><li>The test-running framework</li><li>The application or system under test (SUT) and the services and libraries that the SUT and testing framework depend upon</li><li>The OS and hardware and network that the SUT and testing framework depend upon</li></ul><div><br /></div></div><div>This was captured and summarized in the following diagram.</div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjUnN8lpHRkVYQ3G5ycEcr1HDvMHUAiCsr7nHqYYSIX0_U7A78wjhZrP5CnZLB6eqyjJk_46swSKkL835e_QtWuDZJ_65cL7mn6a_58Bes5Q_Z1_DmNZ7egieAUmlf0zu-FeYNp/s720/Copy+of+Copy+of+Test+Flakiness+Article+II+for+External+Testing+Blog+%25281%2529.jpg" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="472" data-original-width="720" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjUnN8lpHRkVYQ3G5ycEcr1HDvMHUAiCsr7nHqYYSIX0_U7A78wjhZrP5CnZLB6eqyjJk_46swSKkL835e_QtWuDZJ_65cL7mn6a_58Bes5Q_Z1_DmNZ7egieAUmlf0zu-FeYNp/w608-h400/Copy+of+Copy+of+Test+Flakiness+Article+II+for+External+Testing+Blog+%25281%2529.jpg" width="608" /></a></div><div class="separator" style="clear: both; text-align: left;"><div class="separator" style="clear: both;">The reasons, triage tips, and remedies for flakiness are discussed below, by component.</div><div class="separator" style="clear: both;"><br /></div><div class="separator" style="clear: both;"><br /></div><div class="separator" style="clear: both;"><br /></div><div class="separator" style="clear: both;"><h3 style="clear: both; text-align: left;">The tests themselves</h3><div><br /></div><div>The tests themselves can introduce flakiness. This can include test data, test workflows, initial setup of test prerequisites, and initial state of other dependencies.</div><div><br /></div><div><span id="docs-internal-guid-cc3386a8-7fff-514b-d1ff-2a7b704dddcd"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Improper initialization or cleanup.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Look for compiler warnings about uninitialized variables. Inspect initialization and cleanup code. Check that the environment is set up and torn down correctly. Verify that test data is correct.</span></p><br /></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly initialize all variables with proper values before their use.</span></p><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Properly set up and tear down the testing environment. Consider an initial test that verifies the state of the environment.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Invalid assumptions about the state of test data.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Rerun test(s) independently.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Make tests independent of any state from other tests and previous runs.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Invalid assumptions about the state of the system, such as the system time.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly check for system dependency assumptions.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Remove or isolate the SUT dependencies on aspects of the environment that you do not control.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Dependencies on execution time, expecting asynchronous events to occur in a specific order, waiting without timeouts, or race conditions between the tests and the application.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Log the times when accesses to the application are made.</span></p><br /><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">As part of debugging, introduce delays in the application to check for differences in test results.</span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /><br /></span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Add synchronization elements to the tests so that they wait for specific application states. Disable unnecessary caching to have a predictable timeline for the application responses.</span></p><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Note: </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Do</span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">NOT </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">add arbitrary delays as these can become flaky again over time and slow down the test unnecessarily.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Dependencies on the order in which the tests are run. (Similar to the second case above.)</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Rerun test(s) independently.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Make tests independent of each other and of  any state from previous runs.</span></p></td></tr></tbody></table></div></span></div> <div><i><span style="font-size: x-small;"><br /></span></i></div><div><i><span style="font-size: x-small;">Table 1 - Reasons, triage tips, and remedies for flakiness in the tests themselves</span></i></div><div><div><br /></div><h3 style="text-align: left;">The test-running framework</h3></div><div><br /></div><div>An unreliable test-running framework can introduce flakiness. </div><div><br /></div><div><span id="docs-internal-guid-8512e193-7fff-e2f6-818d-6d28c9227926"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Failure to allocate enough resources for the SUT, thus preventing it from running.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check logs to see if SUT came up.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Allocate sufficient resources.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Improper scheduling of the tests so they “collide” and cause each other to fail.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly run tests independently in different order.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Make tests runnable independently of each other.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Insufficient system resources to satisfy the test requirements. (Similar to the first case but here resources are consumed while running the workflow.)</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check system logs to see if SUT ran out of resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix memory leaks or similar resource “bleeding.”</span></p><br /><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Allocate sufficient resources to run tests.</span></p></td></tr></tbody></table></div><br /></span></div><div><i><span style="font-size: x-small;">Table 2 - Reasons, triage tips, and remedies for flakiness in the test running framework</span></i></div><div><br /></div><div><br /></div><h3 style="text-align: left;">The application or SUT and the services and libraries that the SUT and testing framework depend upon</h3><div><div><br /></div><div>Of course, the application itself (or the SUT) could be the source of flakiness. </div><div>An application can also have numerous dependencies on other services, and each of those services can have their own dependencies. In this chain, each of the services can introduce flakiness. </div></div><div><br /></div><div><span id="docs-internal-guid-09658aab-7fff-21ca-2937-a430c1039703"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Race conditions.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Log accesses of shared resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Add synchronization elements to the tests so that they wait for specific application states. Note: </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Do</span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">NOT </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">add arbitrary delays as these can become flaky again over time.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Uninitialized variables.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Look for compiler warnings about uninitialized variables.</span></p><br /></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly initialize all variables with proper values before their use.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Being slow to respond or being unresponsive to the stimuli from the tests.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Log the times when requests and responses are made.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check and remove any causes for delays.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Memory leaks.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Look at memory consumption during test runs. Use tools such as Valgrind to detect.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix programming error causing memory leak. This </span><a href="https://en.wikipedia.org/wiki/Memory_leak" style="text-decoration-line: none;"><span style="color: #1155cc; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; text-decoration-line: underline; text-decoration-skip-ink: none; vertical-align: baseline; white-space: pre-wrap;">Wikipedia article</span></a><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> has an excellent discussion on these types of errors.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Oversubscription of resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check system logs to see if SUT ran out of resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Allocate sufficient resources to run tests.</span></p><br /></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Changes to the application (or dependent services) out of sync with the corresponding tests.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Examine revision history.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Institute a policy requiring code changes to be accompanied by tests.</span></p></td></tr></tbody></table></div><br /></span></div><div><span id="docs-internal-guid-ac8dffbb-7fff-2802-cd50-177a19301a76"><div align="left" dir="ltr" style="margin-left: 0pt;"><div align="left" dir="ltr" style="margin-left: 0pt;"><i><span style="font-size: x-small;">Table 3 - Reasons, triage tips, and remedies for flakiness in the application or SUT</span></i></div><div><br /></div><div><br /></div><div><h3>The OS and hardware that the SUT and testing framework depend upon</h3></div><div><br /></div><div>Finally, the underlying hardware and operating system can be sources of test flakiness. </div><div><i><span style="font-size: x-small;"><br /></span></i></div><div><span id="docs-internal-guid-0d6c36b5-7fff-566e-92ce-795efef74cfe"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Networking failures or instability.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check for hardware errors in system logs.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix hardware errors or run tests on different hardware.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Disk errors.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check for hardware errors in system logs.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix hardware errors or run tests on different hardware.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Resources being consumed by other tasks/services not related to the tests being run.</span></p><br /></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Examine system process activity.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Reduce activity of other processes on test system(s).</span></p></td></tr></tbody></table></div><br /></span></div><div><i><span style="font-size: x-small;">Table 4 - Reasons, triage tips, and remedies for flakiness in the OS and hardware of the SUT</span></i></div><div><br /></div><div><br /></div><h3>Conclusion</h3><div>As can be seen from the wide variety of failures, having low flakiness in automated testing can be quite a challenge. This article has outlined both the components under which tests are run and the types of flakiness that can occur, and thus can serve as a cheat sheet when triaging and fixing flaky tests.</div><div><br /></div><div><br /></div><div><h3>References</h3></div><div><ul><li><a href="https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html">Where do our flaky tests come from?</a></li><li><a href="https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html">Flaky Tests at Google and How We Mitigate Them</a></li><li><a href="https://testing.googleblog.com/2009/06/my-selenium-tests-arent-stable.html">My Selenium Tests Aren't Stable!</a></li><li><a href="https://testing.googleblog.com/2008/04/tott-avoiding-flakey-tests.html">TotT: Avoiding Flakey Tests</a></li><li><a href="https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html">Test Flakiness - One of the main challenges of automated testing</a></li></ul></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div></div></span></div></div></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> By George Pirocanac<div><br /></div><div>This is part two of a series on test flakiness. The <a href="https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html">first article</a> discussed the four components under which tests are run and the possible reasons for test flakiness. This article will discuss the triage tips and remedies for flakiness for each of these possible reasons.</div><h3 style="text-align: left;"><br /></h3><h3 style="text-align: left;">Components</h3><div><br /></div><div>To review, the four components where flakiness can occur include:</div><div><ul style="text-align: left;"><li>The tests themselves</li><li>The test-running framework</li><li>The application or system under test (SUT) and the services and libraries that the SUT and testing framework depend upon</li><li>The OS and hardware and network that the SUT and testing framework depend upon</li></ul><div><br /></div></div><div>This was captured and summarized in the following diagram.</div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjUnN8lpHRkVYQ3G5ycEcr1HDvMHUAiCsr7nHqYYSIX0_U7A78wjhZrP5CnZLB6eqyjJk_46swSKkL835e_QtWuDZJ_65cL7mn6a_58Bes5Q_Z1_DmNZ7egieAUmlf0zu-FeYNp/s720/Copy+of+Copy+of+Test+Flakiness+Article+II+for+External+Testing+Blog+%25281%2529.jpg" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="472" data-original-width="720" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjUnN8lpHRkVYQ3G5ycEcr1HDvMHUAiCsr7nHqYYSIX0_U7A78wjhZrP5CnZLB6eqyjJk_46swSKkL835e_QtWuDZJ_65cL7mn6a_58Bes5Q_Z1_DmNZ7egieAUmlf0zu-FeYNp/w608-h400/Copy+of+Copy+of+Test+Flakiness+Article+II+for+External+Testing+Blog+%25281%2529.jpg" width="608" /></a></div><div class="separator" style="clear: both; text-align: left;"><div class="separator" style="clear: both;">The reasons, triage tips, and remedies for flakiness are discussed below, by component.</div><div class="separator" style="clear: both;"><br /></div><div class="separator" style="clear: both;"><br /></div><div class="separator" style="clear: both;"><br /></div><div class="separator" style="clear: both;"><h3 style="clear: both; text-align: left;">The tests themselves</h3><div><br /></div><div>The tests themselves can introduce flakiness. This can include test data, test workflows, initial setup of test prerequisites, and initial state of other dependencies.</div><div><br /></div><div><span id="docs-internal-guid-cc3386a8-7fff-514b-d1ff-2a7b704dddcd"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Improper initialization or cleanup.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Look for compiler warnings about uninitialized variables. Inspect initialization and cleanup code. Check that the environment is set up and torn down correctly. Verify that test data is correct.</span></p><br /></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly initialize all variables with proper values before their use.</span></p><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Properly set up and tear down the testing environment. Consider an initial test that verifies the state of the environment.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Invalid assumptions about the state of test data.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Rerun test(s) independently.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Make tests independent of any state from other tests and previous runs.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Invalid assumptions about the state of the system, such as the system time.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly check for system dependency assumptions.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Remove or isolate the SUT dependencies on aspects of the environment that you do not control.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Dependencies on execution time, expecting asynchronous events to occur in a specific order, waiting without timeouts, or race conditions between the tests and the application.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Log the times when accesses to the application are made.</span></p><br /><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">As part of debugging, introduce delays in the application to check for differences in test results.</span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /><br /></span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Add synchronization elements to the tests so that they wait for specific application states. Disable unnecessary caching to have a predictable timeline for the application responses.</span></p><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Note: </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Do</span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">NOT </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">add arbitrary delays as these can become flaky again over time and slow down the test unnecessarily.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Dependencies on the order in which the tests are run. (Similar to the second case above.)</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Rerun test(s) independently.</span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Make tests independent of each other and of  any state from previous runs.</span></p></td></tr></tbody></table></div></span></div> <div><i><span style="font-size: x-small;"><br /></span></i></div><div><i><span style="font-size: x-small;">Table 1 - Reasons, triage tips, and remedies for flakiness in the tests themselves</span></i></div><div><div><br /></div><h3 style="text-align: left;">The test-running framework</h3></div><div><br /></div><div>An unreliable test-running framework can introduce flakiness. </div><div><br /></div><div><span id="docs-internal-guid-8512e193-7fff-e2f6-818d-6d28c9227926"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Failure to allocate enough resources for the SUT, thus preventing it from running.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check logs to see if SUT came up.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Allocate sufficient resources.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Improper scheduling of the tests so they “collide” and cause each other to fail.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly run tests independently in different order.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Make tests runnable independently of each other.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Insufficient system resources to satisfy the test requirements. (Similar to the first case but here resources are consumed while running the workflow.)</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check system logs to see if SUT ran out of resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix memory leaks or similar resource “bleeding.”</span></p><br /><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Allocate sufficient resources to run tests.</span></p></td></tr></tbody></table></div><br /></span></div><div><i><span style="font-size: x-small;">Table 2 - Reasons, triage tips, and remedies for flakiness in the test running framework</span></i></div><div><br /></div><div><br /></div><h3 style="text-align: left;">The application or SUT and the services and libraries that the SUT and testing framework depend upon</h3><div><div><br /></div><div>Of course, the application itself (or the SUT) could be the source of flakiness. </div><div>An application can also have numerous dependencies on other services, and each of those services can have their own dependencies. In this chain, each of the services can introduce flakiness. </div></div><div><br /></div><div><span id="docs-internal-guid-09658aab-7fff-21ca-2937-a430c1039703"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Race conditions.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Log accesses of shared resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Add synchronization elements to the tests so that they wait for specific application states. Note: </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Do</span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">NOT </span><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">add arbitrary delays as these can become flaky again over time.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Uninitialized variables.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Look for compiler warnings about uninitialized variables.</span></p><br /></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Explicitly initialize all variables with proper values before their use.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Being slow to respond or being unresponsive to the stimuli from the tests.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Log the times when requests and responses are made.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check and remove any causes for delays.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Memory leaks.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Look at memory consumption during test runs. Use tools such as Valgrind to detect.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix programming error causing memory leak. This </span><a href="https://en.wikipedia.org/wiki/Memory_leak" style="text-decoration-line: none;"><span style="color: #1155cc; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; text-decoration-line: underline; text-decoration-skip-ink: none; vertical-align: baseline; white-space: pre-wrap;">Wikipedia article</span></a><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"> has an excellent discussion on these types of errors.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Oversubscription of resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check system logs to see if SUT ran out of resources.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Allocate sufficient resources to run tests.</span></p><br /></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Changes to the application (or dependent services) out of sync with the corresponding tests.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Examine revision history.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Institute a policy requiring code changes to be accompanied by tests.</span></p></td></tr></tbody></table></div><br /></span></div><div><span id="docs-internal-guid-ac8dffbb-7fff-2802-cd50-177a19301a76"><div align="left" dir="ltr" style="margin-left: 0pt;"><div align="left" dir="ltr" style="margin-left: 0pt;"><i><span style="font-size: x-small;">Table 3 - Reasons, triage tips, and remedies for flakiness in the application or SUT</span></i></div><div><br /></div><div><br /></div><div><h3>The OS and hardware that the SUT and testing framework depend upon</h3></div><div><br /></div><div>Finally, the underlying hardware and operating system can be sources of test flakiness. </div><div><i><span style="font-size: x-small;"><br /></span></i></div><div><span id="docs-internal-guid-0d6c36b5-7fff-566e-92ce-795efef74cfe"><br /><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none; table-layout: fixed; width: 468pt;"><colgroup><col></col><col></col><col></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Reason for Flakiness</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Tips for Triaging</span></p></td><td style="background-color: #cfe2f3; border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="background-color: transparent; font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">Type of Remedy</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Networking failures or instability.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check for hardware errors in system logs.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix hardware errors or run tests on different hardware.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Disk errors.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Check for hardware errors in system logs.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Fix hardware errors or run tests on different hardware.</span></p></td></tr><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Resources being consumed by other tasks/services not related to the tests being run.</span></p><br /></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Examine system process activity.</span></p></td><td style="border-bottom: solid #000000 1pt; border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-top: solid #000000 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt 5pt 5pt 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: Arial; font-size: 11pt; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">Reduce activity of other processes on test system(s).</span></p></td></tr></tbody></table></div><br /></span></div><div><i><span style="font-size: x-small;">Table 4 - Reasons, triage tips, and remedies for flakiness in the OS and hardware of the SUT</span></i></div><div><br /></div><div><br /></div><h3>Conclusion</h3><div>As can be seen from the wide variety of failures, having low flakiness in automated testing can be quite a challenge. This article has outlined both the components under which tests are run and the types of flakiness that can occur, and thus can serve as a cheat sheet when triaging and fixing flaky tests.</div><div><br /></div><div><br /></div><div><h3>References</h3></div><div><ul><li><a href="https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html">Where do our flaky tests come from?</a></li><li><a href="https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html">Flaky Tests at Google and How We Mitigate Them</a></li><li><a href="https://testing.googleblog.com/2009/06/my-selenium-tests-arent-stable.html">My Selenium Tests Aren't Stable!</a></li><li><a href="https://testing.googleblog.com/2008/04/tott-avoiding-flakey-tests.html">TotT: Avoiding Flakey Tests</a></li><li><a href="https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html">Test Flakiness - One of the main challenges of automated testing</a></li></ul></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div><div><br /></div></div></span></div></div></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Test Flakiness - One of the main challenges of automated testing (Part II)&url=https://testing.googleblog.com/2021/03/test-flakiness-one-of-main-challenges.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2021/03/test-flakiness-one-of-main-challenges.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2021/03/test-flakiness-one-of-main-challenges.html#comments' style='font-weight: 500; text-decoration: underline;'>3 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2021/03/test-flakiness-one-of-main-challenges.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/George%20Pirocanac' rel='tag'> George Pirocanac </a> , <a class='label' href='https://testing.googleblog.com/search/label/Test%20Flakiness' rel='tag'> Test Flakiness </a> </span> </div> </div> </div> <div class='post' data-id='7086735822667839051' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html' itemprop='url' title='Test Flakiness - One of the main challenges of automated testing'> Test Flakiness - One of the main challenges of automated testing </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Wednesday, December 16, 2020 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By George Pirocanac<div><br /></div><div><br /></div><div>Dealing with test flakiness is a critical skill in testing because automated tests that do not provide a consistent signal will slow down the entire development process. If you haven’t encountered flaky tests, this article is a must-read as it first tries to systematically outline the causes for flaky tests. If you have encountered flaky tests, see how many fall into the areas listed.</div><div><br /></div><div><br /></div><div><div>A follow-up article will talk about dealing with each of the causes.</div></div><div><br /></div><div><br /></div><div>Over the years I’ve seen a lot of reasons for flaky tests, but rather than review them one by one, let’s group the sources of flakiness by the components under which tests are run:</div><div><ul style="text-align: left;"><li>The tests themselves</li><li>The test-running framework</li><li>The application or system under Test (SUT) and the services and libraries that the SUT and testing framework depend upon</li><li>The OS and hardware that the SUT and testing framework depend upon</li></ul><div><br /></div></div><div><div>This is illustrated below. Figure 1 first shows the hardware/software stack that supports an application or system under test. At the lowest level is the hardware. The next level up is the operating system followed by the libraries that provide an interface to the system. At the highest level, is the middleware, the layer that provides application specific interfaces.</div></div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiL7-sd5koCPql8mJeq-5b4Vjo8Ex1nWHlihjQ2w-0AHXbIZLWb0cL9zCuqzkoV0RspcxQTtXE_JSmJHk6nStrW7NUxQ8lvHEzCBw5dchj1Jowg4EHYb9G6U-ZyiOGsQIE3rzW9/s460/Test+Flakiness+-+Figure+1+%25281%2529.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="460" data-original-width="439" height="346" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiL7-sd5koCPql8mJeq-5b4Vjo8Ex1nWHlihjQ2w-0AHXbIZLWb0cL9zCuqzkoV0RspcxQTtXE_JSmJHk6nStrW7NUxQ8lvHEzCBw5dchj1Jowg4EHYb9G6U-ZyiOGsQIE3rzW9/w330-h346/Test+Flakiness+-+Figure+1+%25281%2529.jpg" width="330" /></a></div><div class="separator" style="clear: both; text-align: center;"><br /><br /></div><div>In a distributed system, however, each of the services of the application and the services it depends upon can reside on a different hardware / software stack as can the test running service. This is illustrated in Figure 2 as the full test running environment.</div><div><br /></div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgQLc8lZ3LethyhQkmibpBwkEqNMVvjznwN7b5MhDimx1AlC-QKtuvpfaGs6tUhuQJ715klFCvqzCjbT8juhTQT0ccn-eSfS2dCuekKvxK-bXBVv7w7PA2VCY1X0hfgAVbXn6k3/s726/Test+Flakiness+-+Figure+2+%25281%2529.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="520" data-original-width="726" height="392" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgQLc8lZ3LethyhQkmibpBwkEqNMVvjznwN7b5MhDimx1AlC-QKtuvpfaGs6tUhuQJ715klFCvqzCjbT8juhTQT0ccn-eSfS2dCuekKvxK-bXBVv7w7PA2VCY1X0hfgAVbXn6k3/w548-h392/Test+Flakiness+-+Figure+2+%25281%2529.jpg" width="548" /></a></div><div class="separator" style="clear: both; text-align: center;"><br /></div><div><br /></div><div><div>As discussed above, each of these components is a potential area for flakiness.</div></div><div><br /></div><div><br /></div><div><h3 style="text-align: left;">The tests themselves</h3></div><div><br /></div><div><div>The tests themselves can introduce flakiness. Typical causes include:</div></div><div><ul style="text-align: left;"><li>Improper initialization or cleanup.</li><li>Invalid assumptions about the state of test data.</li><li>Invalid assumptions about the state of the system. An example can be the system time.</li><li>Dependencies on the timing of the application.</li><li>Dependencies on the order in which the tests are run. (Similar to the first case above.)</li></ul></div><div><br /></div><div><br /></div><h3 style="text-align: left;">The test-running framework</h3><div><br /></div><div>An unreliable test-running framework can introduce flakiness. Typical causes include:</div><div><br /></div><div><ul style="text-align: left;"><li>Failure to allocate enough resources for the system under test thus causing it to fail coming up. </li><li>Improper scheduling of the tests so they “collide” and cause each other to fail.</li><li>Insufficient system resources to satisfy the test requirements.</li></ul></div><div><br /></div><h3 style="text-align: left;">The application or system under test and the services and libraries that the SUT and testing framework depend upon</h3><div><br /></div><div><div>Of course, the application itself (or the system under test) could be the source of flakiness. An application can also have numerous dependencies on other services, and each of those services can have their own dependencies. In this chain, each of the services can introduce flakiness. Typical causes include:</div></div><div><ul style="text-align: left;"><li>Race conditions.</li><li>Uninitialized variables.</li><li>Being slow to respond or being unresponsive to the stimuli from the tests.</li><li>Memory leaks.</li><li>Oversubscription of resources.</li><li>Changes to the application (or dependent services) happening at a different pace than those to the corresponding tests.</li></ul></div><div><br /></div><div>Testing environments are called <i><a href="https://testing.googleblog.com/2012/10/hermetic-servers.html">hermetic</a></i> when they contain everything that is needed to run the tests (i.e. no external dependencies like servers running in production). Hermetic environments, in general, are less likely to be flaky.</div><div><br /></div><h3 style="text-align: left;">The OS and hardware that the SUT and testing framework depend upon</h3><div><br /></div><div><br /></div><div>Finally, the underlying hardware and operating system can be the source of test flakiness. Typical causes include:</div><div><ul style="text-align: left;"><li>Networking failures or instability.</li><li>Disk errors.</li><li>Resources being consumed by other tasks/services not related to the tests being run.</li></ul></div><div><br /></div><div><div>As can be seen from the wide variety of failures, having low flakiness in automated testing can be quite a challenge. This article has both outlined the areas and the types of flakiness that can occur in those areas, so it can serve as a cheat sheet when triaging flaky tests.</div><div><br /></div><div><br /></div><div>In the follow-up of this blog we’ll look at ways of addressing these issues.</div></div><div><br /></div><h3 style="text-align: left;"><br /></h3><h3 style="text-align: left;">References</h3><div><div><ul style="text-align: left;"><li><a href="https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html">Where do our flaky tests come from?</a></li><li><a href="https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html">Flaky Tests at Google and How We Mitigate Them</a></li><li><a href="https://testing.googleblog.com/2009/06/my-selenium-tests-arent-stable.html">My Selenium Tests Aren't Stable!</a></li><li><a href="https://testing.googleblog.com/2008/04/tott-avoiding-flakey-tests.html">TotT: Avoiding Flakey Tests</a></li></ul></div></div><div><br /></div><div><br /></div><div><br /></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> By George Pirocanac<div><br /></div><div><br /></div><div>Dealing with test flakiness is a critical skill in testing because automated tests that do not provide a consistent signal will slow down the entire development process. If you haven’t encountered flaky tests, this article is a must-read as it first tries to systematically outline the causes for flaky tests. If you have encountered flaky tests, see how many fall into the areas listed.</div><div><br /></div><div><br /></div><div><div>A follow-up article will talk about dealing with each of the causes.</div></div><div><br /></div><div><br /></div><div>Over the years I’ve seen a lot of reasons for flaky tests, but rather than review them one by one, let’s group the sources of flakiness by the components under which tests are run:</div><div><ul style="text-align: left;"><li>The tests themselves</li><li>The test-running framework</li><li>The application or system under Test (SUT) and the services and libraries that the SUT and testing framework depend upon</li><li>The OS and hardware that the SUT and testing framework depend upon</li></ul><div><br /></div></div><div><div>This is illustrated below. Figure 1 first shows the hardware/software stack that supports an application or system under test. At the lowest level is the hardware. The next level up is the operating system followed by the libraries that provide an interface to the system. At the highest level, is the middleware, the layer that provides application specific interfaces.</div></div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiL7-sd5koCPql8mJeq-5b4Vjo8Ex1nWHlihjQ2w-0AHXbIZLWb0cL9zCuqzkoV0RspcxQTtXE_JSmJHk6nStrW7NUxQ8lvHEzCBw5dchj1Jowg4EHYb9G6U-ZyiOGsQIE3rzW9/s460/Test+Flakiness+-+Figure+1+%25281%2529.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="460" data-original-width="439" height="346" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiL7-sd5koCPql8mJeq-5b4Vjo8Ex1nWHlihjQ2w-0AHXbIZLWb0cL9zCuqzkoV0RspcxQTtXE_JSmJHk6nStrW7NUxQ8lvHEzCBw5dchj1Jowg4EHYb9G6U-ZyiOGsQIE3rzW9/w330-h346/Test+Flakiness+-+Figure+1+%25281%2529.jpg" width="330" /></a></div><div class="separator" style="clear: both; text-align: center;"><br /><br /></div><div>In a distributed system, however, each of the services of the application and the services it depends upon can reside on a different hardware / software stack as can the test running service. This is illustrated in Figure 2 as the full test running environment.</div><div><br /></div><div><br /></div><div class="separator" style="clear: both; text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgQLc8lZ3LethyhQkmibpBwkEqNMVvjznwN7b5MhDimx1AlC-QKtuvpfaGs6tUhuQJ715klFCvqzCjbT8juhTQT0ccn-eSfS2dCuekKvxK-bXBVv7w7PA2VCY1X0hfgAVbXn6k3/s726/Test+Flakiness+-+Figure+2+%25281%2529.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" data-original-height="520" data-original-width="726" height="392" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgQLc8lZ3LethyhQkmibpBwkEqNMVvjznwN7b5MhDimx1AlC-QKtuvpfaGs6tUhuQJ715klFCvqzCjbT8juhTQT0ccn-eSfS2dCuekKvxK-bXBVv7w7PA2VCY1X0hfgAVbXn6k3/w548-h392/Test+Flakiness+-+Figure+2+%25281%2529.jpg" width="548" /></a></div><div class="separator" style="clear: both; text-align: center;"><br /></div><div><br /></div><div><div>As discussed above, each of these components is a potential area for flakiness.</div></div><div><br /></div><div><br /></div><div><h3 style="text-align: left;">The tests themselves</h3></div><div><br /></div><div><div>The tests themselves can introduce flakiness. Typical causes include:</div></div><div><ul style="text-align: left;"><li>Improper initialization or cleanup.</li><li>Invalid assumptions about the state of test data.</li><li>Invalid assumptions about the state of the system. An example can be the system time.</li><li>Dependencies on the timing of the application.</li><li>Dependencies on the order in which the tests are run. (Similar to the first case above.)</li></ul></div><div><br /></div><div><br /></div><h3 style="text-align: left;">The test-running framework</h3><div><br /></div><div>An unreliable test-running framework can introduce flakiness. Typical causes include:</div><div><br /></div><div><ul style="text-align: left;"><li>Failure to allocate enough resources for the system under test thus causing it to fail coming up. </li><li>Improper scheduling of the tests so they “collide” and cause each other to fail.</li><li>Insufficient system resources to satisfy the test requirements.</li></ul></div><div><br /></div><h3 style="text-align: left;">The application or system under test and the services and libraries that the SUT and testing framework depend upon</h3><div><br /></div><div><div>Of course, the application itself (or the system under test) could be the source of flakiness. An application can also have numerous dependencies on other services, and each of those services can have their own dependencies. In this chain, each of the services can introduce flakiness. Typical causes include:</div></div><div><ul style="text-align: left;"><li>Race conditions.</li><li>Uninitialized variables.</li><li>Being slow to respond or being unresponsive to the stimuli from the tests.</li><li>Memory leaks.</li><li>Oversubscription of resources.</li><li>Changes to the application (or dependent services) happening at a different pace than those to the corresponding tests.</li></ul></div><div><br /></div><div>Testing environments are called <i><a href="https://testing.googleblog.com/2012/10/hermetic-servers.html">hermetic</a></i> when they contain everything that is needed to run the tests (i.e. no external dependencies like servers running in production). Hermetic environments, in general, are less likely to be flaky.</div><div><br /></div><h3 style="text-align: left;">The OS and hardware that the SUT and testing framework depend upon</h3><div><br /></div><div><br /></div><div>Finally, the underlying hardware and operating system can be the source of test flakiness. Typical causes include:</div><div><ul style="text-align: left;"><li>Networking failures or instability.</li><li>Disk errors.</li><li>Resources being consumed by other tasks/services not related to the tests being run.</li></ul></div><div><br /></div><div><div>As can be seen from the wide variety of failures, having low flakiness in automated testing can be quite a challenge. This article has both outlined the areas and the types of flakiness that can occur in those areas, so it can serve as a cheat sheet when triaging flaky tests.</div><div><br /></div><div><br /></div><div>In the follow-up of this blog we’ll look at ways of addressing these issues.</div></div><div><br /></div><h3 style="text-align: left;"><br /></h3><h3 style="text-align: left;">References</h3><div><div><ul style="text-align: left;"><li><a href="https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html">Where do our flaky tests come from?</a></li><li><a href="https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html">Flaky Tests at Google and How We Mitigate Them</a></li><li><a href="https://testing.googleblog.com/2009/06/my-selenium-tests-arent-stable.html">My Selenium Tests Aren't Stable!</a></li><li><a href="https://testing.googleblog.com/2008/04/tott-avoiding-flakey-tests.html">TotT: Avoiding Flakey Tests</a></li></ul></div></div><div><br /></div><div><br /></div><div><br /></div> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Test Flakiness - One of the main challenges of automated testing&url=https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html#comments' style='font-weight: 500; text-decoration: underline;'>9 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2020/12/test-flakiness-one-of-main-challenges.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/George%20Pirocanac' rel='tag'> George Pirocanac </a> , <a class='label' href='https://testing.googleblog.com/search/label/Test%20Flakiness' rel='tag'> Test Flakiness </a> </span> </div> </div> </div> <div class='blog-pager' id='blog-pager'> <a class='home-link' href='https://testing.googleblog.com/'> <i class='material-icons'>  </i> </a> <i class='material-icons disabled'>  </i> <span id='blog-pager-older-link'> <a class='blog-pager-older-link' href='https://testing.googleblog.com/search/label/Test%20Flakiness?updated-max=2020-12-16T09:32:00-08:00&max-results=20&start=20&by-date=false' id='Blog1_blog-pager-older-link' title='Older Posts'> <i class='material-icons'>  </i> </a> </span> </div> <div class='clear'></div> </div></div> </div> </div> <div class='col-right'> <div class='section' id='sidebar-top'><div class='widget HTML' data-version='1' id='HTML8'> <div class='widget-content'> <div class='searchBox'> <input type='text' title='Search This Blog' placeholder='Search blog ...' /> </div> </div> <div class='clear'></div> </div> </div> <div id='aside'> <div class='section' id='sidebar'><div class='widget Label' data-version='1' id='Label1'> <div class='tab'> <img class='sidebar-icon' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII='/> <h2> Labels </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content list-label-widget-content'> <ul> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/TotT'> TotT </a> <span dir='ltr'> 102 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/GTAC'> GTAC </a> <span dir='ltr'> 61 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/James%20Whittaker'> James Whittaker </a> <span dir='ltr'> 42 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Misko%20Hevery'> Misko Hevery </a> <span dir='ltr'> 32 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Code%20Health'> Code Health </a> <span dir='ltr'> 30 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anthony%20Vallone'> Anthony Vallone </a> <span dir='ltr'> 27 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrick%20Copeland'> Patrick Copeland </a> <span dir='ltr'> 23 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jobs'> Jobs </a> <span dir='ltr'> 18 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Andrew%20Trenk'> Andrew Trenk </a> <span dir='ltr'> 12 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/C%2B%2B'> C++ </a> <span dir='ltr'> 11 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund'> Patrik Höglund </a> <span dir='ltr'> 8 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/JavaScript'> JavaScript </a> <span dir='ltr'> 7 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Allen%20Hutchison'> Allen Hutchison </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/George%20Pirocanac'> George Pirocanac </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zhanyong%20Wan'> Zhanyong Wan </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Harry%20Robinson'> Harry Robinson </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Java'> Java </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julian%20Harty'> Julian Harty </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Bender'> Adam Bender </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alberto%20Savoia'> Alberto Savoia </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ben%20Yu'> Ben Yu </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Erik%20Kuefler'> Erik Kuefler </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Philip%20Zembrod'> Philip Zembrod </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shyam%20Seshadri'> Shyam Seshadri </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome'> Chrome </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dillon%20Bly'> Dillon Bly </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Thomas'> John Thomas </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Lesley%20Katzen'> Lesley Katzen </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Kaplan'> Marc Kaplan </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Markus%20Clermont'> Markus Clermont </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander'> Max Kanat-Alexander </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sonal%20Shah'> Sonal Shah </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/APIs'> APIs </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Abhishek%20Arya'> Abhishek Arya </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Myrvold'> Alan Myrvold </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alek%20Icev'> Alek Icev </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Android'> Android </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/April%20Fools'> April Fools </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chaitali%20Narla'> Chaitali Narla </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chris%20Lewis'> Chris Lewis </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome%20OS'> Chrome OS </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Salas'> Diego Salas </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dori%20Reuveni'> Dori Reuveni </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Arbon'> Jason Arbon </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jochen%20Wuttke'> Jochen Wuttke </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kostya%20Serebryany'> Kostya Serebryany </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Eaddy'> Marc Eaddy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87'> Marko Ivanković </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mobile'> Mobile </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Oliver%20Chang'> Oliver Chang </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simon%20Stewart'> Simon Stewart </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stefan%20Kennedy'> Stefan Kennedy </a> <span dir='ltr'> 2 </span> </li> <li> <span dir='ltr'> Test Flakiness </span> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Titus%20Winters'> Titus Winters </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tony%20Voellm'> Tony Voellm </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/WebRTC'> WebRTC </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yiming%20Sun'> Yiming Sun </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yvette%20Nameth'> Yvette Nameth </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zuri%20Kemp'> Zuri Kemp </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Aaron%20Jacobs'> Aaron Jacobs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Porter'> Adam Porter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Raider'> Adam Raider </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adel%20Saoud'> Adel Saoud </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Faulkner'> Alan Faulkner </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alex%20Eagle'> Alex Eagle </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Amy%20Fu'> Amy Fu </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anantha%20Keesara'> Anantha Keesara </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Antoine%20Picard'> Antoine Picard </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/App%20Engine'> App Engine </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ari%20Shamash'> Ari Shamash </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Arif%20Sukoco'> Arif Sukoco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Benjamin%20Pick'> Benjamin Pick </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bob%20Nystrom'> Bob Nystrom </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bruce%20Leban'> Bruce Leban </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Arguelles'> Carlos Arguelles </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Israel%20Ortiz%20Garc%C3%ADa'> Carlos Israel Ortiz García </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Cathal%20Weakliam'> Cathal Weakliam </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Christopher%20Semturs'> Christopher Semturs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Clay%20Murphy'> Clay Murphy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dagang%20Wei'> Dagang Wei </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Maksimovich'> Dan Maksimovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Shi'> Dan Shi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Willemsen'> Dan Willemsen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Chen'> Dave Chen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Gladfelter'> Dave Gladfelter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Bendory'> David Bendory </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Mandelberg'> David Mandelberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Derek%20Snyder'> Derek Snyder </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Cavalcanti'> Diego Cavalcanti </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov'> Dmitry Vyukov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz'> Eduardo Bravo Ortiz </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ekaterina%20Kamenskaya'> Ekaterina Kamenskaya </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliott%20Karpilovsky'> Elliott Karpilovsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliotte%20Rusty%20Harold'> Elliotte Rusty Harold </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Espresso'> Espresso </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Felipe%20Sodr%C3%A9'> Felipe Sodré </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Francois%20Aube'> Francois Aube </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Gene%20Volovich'> Gene Volovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Google%2B'> Google+ </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goran%20Petrovic'> Goran Petrovic </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goranka%20Bjedov'> Goranka Bjedov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hank%20Duan'> Hank Duan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Havard%20Rast%20Blok'> Havard Rast Blok </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hongfei%20Ding'> Hongfei Ding </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Elbaum'> Jason Elbaum </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Huggins'> Jason Huggins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jay%20Han'> Jay Han </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Hoy'> Jeff Hoy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Listfield'> Jeff Listfield </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jessica%20Tomechak'> Jessica Tomechak </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jim%20Reardon'> Jim Reardon </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joe%20Allan%20Muharsky'> Joe Allan Muharsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joel%20Hynoski'> Joel Hynoski </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Micco'> John Micco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Penix'> John Penix </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway'> Jonathan Rockway </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez'> Jonathan Velasquez </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Josh%20Armour'> Josh Armour </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julie%20Ralph'> Julie Ralph </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kai%20Kent'> Kai Kent </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Karin%20Lundberg'> Karin Lundberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kaue%20Silveira'> Kaue Silveira </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Bourrillion'> Kevin Bourrillion </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Graney'> Kevin Graney </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kirkland'> Kirkland </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever'> Kurt Alfred Kluever </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Manjusha%20Parvathaneni'> Manjusha Parvathaneni </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis'> Marek Kiszkis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marius%20Latinis'> Marius Latinis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Ivey'> Mark Ivey </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Manley'> Mark Manley </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Striebeck'> Mark Striebeck </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Matt%20Lowrie'> Matt Lowrie </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Meredith%20Whittaker'> Meredith Whittaker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Bachman'> Michael Bachman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Klepikov'> Michael Klepikov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Aizatsky'> Mike Aizatsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Wacker'> Mike Wacker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy'> Mona El Mahdy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Noel%20Yap'> Noel Yap </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Palak%20Bansal'> Palak Bansal </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patricia%20Legaspi'> Patricia Legaspi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Per%20Jacobsson'> Per Jacobsson </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht'> Peter Arrenbrecht </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Spragins'> Peter Spragins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Norman'> Phil Norman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Rollet'> Phil Rollet </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Pooja%20Gupta'> Pooja Gupta </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Project%20Showcase'> Project Showcase </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Radoslav%20Vasilev'> Radoslav Vasilev </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Dewan'> Rajat Dewan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Jain'> Rajat Jain </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rich%20Martin'> Rich Martin </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Richard%20Bustamante'> Richard Bustamante </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Roshan%20Sembacuttiaratchy'> Roshan Sembacuttiaratchy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov'> Ruslan Khamitov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sam%20Lee'> Sam Lee </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sean%20Jordan'> Sean Jordan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sharon%20Zhou'> Sharon Zhou </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shiva%20Garg'> Shiva Garg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Siddartha%20Janga'> Siddartha Janga </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simran%20Basi'> Simran Basi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stan%20Chan'> Stan Chan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stephen%20Ng'> Stephen Ng </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tejas%20Shah'> Tejas Shah </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Analytics'> Test Analytics </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Engineer'> Test Engineer </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tim%20Lyakhovetskiy'> Tim Lyakhovetskiy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tom%20O%27Neill'> Tom O'Neill </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Vojta%20J%C3%ADna'> Vojta Jína </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/automation'> automation </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/dead%20code'> dead code </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/iOS'> iOS </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/mutation%20testing'> mutation testing </a> <span dir='ltr'> 1 </span> </li> </ul> <div class='clear'></div> </div> </div><div class='widget BlogArchive' data-version='1' id='BlogArchive1'> <div class='tab'> <i class='material-icons icon'>  </i> <h2> Archive </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content'> <div id='ArchiveList'> <div id='BlogArchive1_ArchiveList'> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/'> 2024 </a> <span class='post-count' dir='ltr'>(12)</span> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2024/10/smurf-beyond-test-pyramid.html'> SMURF: Beyond the Test Pyramid </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/'> 2023 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/09/'> Sep </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/'> 2022 </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/'> 2021 </a> <span class='post-count' dir='ltr'>(3)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/'> 2020 </a> <span class='post-count' dir='ltr'>(8)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/'> 2019 </a> <span class='post-count' dir='ltr'>(4)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/'> 2018 </a> <span class='post-count' dir='ltr'>(7)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/'> 2017 </a> <span class='post-count' dir='ltr'>(17)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/'> 2016 </a> <span class='post-count' dir='ltr'>(15)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/'> 2015 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/'> 2014 </a> <span class='post-count' dir='ltr'>(24)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/06/'> Jun </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/'> 2013 </a> <span class='post-count' dir='ltr'>(16)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/'> 2012 </a> <span class='post-count' dir='ltr'>(11)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/'> 2011 </a> <span class='post-count' dir='ltr'>(39)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/06/'> Jun </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/05/'> May </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/01/'> Jan </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/'> 2010 </a> <span class='post-count' dir='ltr'>(37)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/11/'> Nov </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/10/'> Oct </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/03/'> Mar </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/'> 2009 </a> <span class='post-count' dir='ltr'>(54)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/07/'> Jul </a> <span class='post-count' dir='ltr'>(15)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/06/'> Jun </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/'> 2008 </a> <span class='post-count' dir='ltr'>(75)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/12/'> Dec </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/11/'> Nov </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/10/'> Oct </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/08/'> Aug </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/07/'> Jul </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/06/'> Jun </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/05/'> May </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/04/'> Apr </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/02/'> Feb </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/'> 2007 </a> <span class='post-count' dir='ltr'>(41)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/10/'> Oct </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/04/'> Apr </a> <span class='post-count' dir='ltr'>(7)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/03/'> Mar </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> </div> </div> <div class='clear'></div> </div> </div><div class='widget HTML' data-version='1' id='HTML6'> <div class='widget-content'> <a href="http://googletesting.blogspot.com/atom.xml"> <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==" class="sidebar-icon" /> <h2>Feed</h2> </a> </div> <div class='clear'></div> </div></div> <div class='section' id='sidebar-bottom'><div class='widget HTML' data-version='1' id='HTML9'> <div class='widget-content'> <a href='http://cloud.feedly.com/#subscription%2Ffeed%2Fhttp%3A%2F%2Fgoogletesting.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='blank'><img id="feedlyFollow" src="https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_vOmq8vkS7pRzxaa3m_gaTsJRiIkixp8IGQm3pO1bzx5o4xY3KMVkEspa_JCYhOe7sgmngXl85Q91527IS4GJUWWJFsnZGGvJDyo1AjDFmJ9dOBNqMTMcD4_Uy2iggwdTJ_6vPrJsRvlWJyWsCtnj8=s0-d" alt="follow us in feedly" width="66" height="20"></a> <div class="share followgooglewrapper"> <button data-href="https://twitter.com/intent/follow?original_referer=http://googletesting.blogspot.com/&screen_name=googletesting" onclick='sharingPopup(this);' id='twitter-share'><span class="twitter-follow">Follow @googletesting</span></button> <script> function sharingPopup (button) { var url = button.getAttribute("data-href"); window.open( url,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); } </script> </div> </div> <div class='clear'></div> </div></div> </div> </div> <div style='clear:both;'></div> </div>  <div class='google-footer-outer loading'> <div id='google-footer'> <a href='//www.google.com/'> <img class='google-logo-dark' height='36' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==' style='margin-top: -16px;' width='92'/> </a> <ul> <li> <a href='//www.google.com/'> Google </a> </li> <li> <a href='//www.google.com/policies/privacy/'> Privacy </a> </li> <li> <a href='//www.google.com/policies/terms/'> Terms </a> </li> </ul> </div> </div> <script type='text/javascript'> //<![CDATA[ // Social sharing popups. var postEl = document.getElementsByClassName('social-wrapper'); var postCount = postEl.length; for(i=0; i<postCount;i++){ postEl[i].addEventListener("click", function(event){ var postUrl = this.getAttribute("data-href"); window.open( postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); });} //]]> </script> <script type='text/javascript'> //<![CDATA[ var BreakpointHandler = function() { this.initted = false; this.isHomePage = false; this.isMobile = false; }; BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) { // Use $.trim for IE8 compatibility summaryHtml = $.trim(summaryHtml).replace(/(<br>|\s)+$/,''); if (lastNode.nodeType == 3) { var lastChar = summaryHtml.slice(-1); if (!lastChar.match(/[.”"?]/)) { if (!lastChar.match(/[A-Za-z]/)) { summaryHtml = summaryHtml.slice(0, -1); } summaryHtml += ' ...'; } } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) { summaryHtml += ' ...'; } return summaryHtml; }; BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) { var seenWords = 0; var summaryHtml = ''; for (var i=0; i < content.childNodes.length; i++) { var node = content.childNodes[i]; var nodeText; if (node.nodeType == 1) { if (node.hasAttribute('data-about-pullquote')) { continue; } nodeText = node.textContent; if (nodeText === undefined) { // innerText for IE8 nodeText = node.innerText; } if (node.nodeName == 'DIV' || node.nodeName == 'B') { // Don't end early if we haven't seen enough words. if (seenWords < 10) { continue; } if (i > 0) { summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]); } break; } summaryHtml += node.outerHTML; } else if (node.nodeType == 3) { nodeText = node.nodeValue; summaryHtml += nodeText + ' '; } var words = nodeText.match(/\S+\s*/g); if (!words) { continue; } var remain = numWords - seenWords; if (words.length >= remain) { summaryHtml = this.finalizeSummary(summaryHtml, node); break; } seenWords += words.length; } return summaryHtml; }; BreakpointHandler.prototype.detect = function() { var match, pl = /\+/g, search = /([^&=]+)=?([^&]*)/g, decode = function (s) { return decodeURIComponent(s.replace(pl, " ")); }, query = window.location.search.substring(1); var urlParams = {}; while (match = search.exec(query)) urlParams[decode(match[1])] = decode(match[2]); this.isListPage = $('html').hasClass('list-page'); this.isMobile = urlParams['m'] === '1'; this.isHomePage = window.location.pathname == '/'; }; BreakpointHandler.prototype.initContent = function() { var self = this; $('.post').each(function(index) { var body = $(this).children('.post-body')[0]; var content = $(body).children('.post-content')[0]; $(content).addClass('post-original'); var data = $(content).children('script').html(); data = self.rewriteForSSL(data); // If exists, extract specified editor's preview. var match = data.match(/([\s\S]+?)<div data-is-preview.+?>([\s\S]+)<\/div>/m); if (match) { data = match[1]; } // Prevent big images from loading when they aren't needed. // This must be done as a pre-injection step, since image loading can't be // canceled once embedded into the DOM. if (self.isListPage && self.isMobile) { data = data.replace(/<(img|iframe) .+?>/g, ''); } // Insert template to be rendered as nodes. content.innerHTML = data; if (self.isListPage) { var summary = document.createElement('div'); $(summary).addClass('post-content'); $(summary).addClass('post-summary'); body.insertBefore(summary, content); if (match) { // Use provided summary. summary.innerHTML = match[2]; } else { // Generate a summary. // Summary generation relies on DOM, so it must occur after content is // inserted into the page. summary.innerHTML = self.generateSummaryFromContent(content, 30); } // Add read more link to summary. var titleAnchor = $(this).find('.title a')[0]; var link = titleAnchor.cloneNode(true); link.innerHTML = 'Read More'; $(link).addClass('read-more'); summary.appendChild(link); } }); // Firefox does not allow for proper styling of BR. if (navigator.userAgent.indexOf('Firefox') > -1) { $('.post-content br').replaceWith('<span class="space"></span>'); } $('.loading').removeClass('loading'); }; BreakpointHandler.prototype.process = function() { if (!this.initted) { var makeInsecureImageRegex = function(hosts) { var whitelist = hosts.join('|').replace(/\./g,'\\.'); // Normal image tags, plus input images (yes, this is possible!) return new RegExp('(<(img|input)[^>]+?src=("|\'))http:\/\/(' + whitelist +')', 'g'); }; this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS); this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]); this.detect(); this.initContent(); this.initted = true; } }; BreakpointHandler.KNOWN_HTTPS_HOSTS = [ "www.google.org", "www.google.com", "services.google.com", "blogger.com", "draft.blogger.com", "www.blogger.com", "photos1.blogger.com", "photos2.blogger.com", "photos3.blogger.com", "blogblog.com", "img1.blogblog.com", "img2.blogblog.com", "www.blogblog.com", "www1.blogblog.com", "www2.blogblog.com", "0.bp.blogspot.com", "1.bp.blogspot.com", "2.bp.blogspot.com", "3.bp.blogspot.com", "4.bp.blogspot.com", "lh3.googleusercontent.com", "lh4.googleusercontent.com", "lh5.googleusercontent.com", "lh6.googleusercontent.com", "themes.googleusercontent.com", ]; BreakpointHandler.prototype.rewriteForSSL = function(html) { // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content. return html.replace(this.sslImageRegex, '$1https://$4') .replace(this.sslImageCurrentDomainRegex, '$1//$4') .replace(/(<(embed|iframe)[^>]+?src=("|'))http:\/\/([^"']*?(youtube|picasaweb\.google)\.com)/g, '$1https://$4') // Slideshow SWF takes a image host, so we need to rewrite that parameter. .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s'); }; $(document).ready(function() { var handler = new BreakpointHandler(); handler.process(); // Top-level navigation. $(".BlogArchive .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); $(".Label .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); // Blog archive year expansion. $('.BlogArchive .intervalToggle').click(function(ev) { ev.preventDefault(); if ($(this).parent().hasClass('collapsed')) { $(this).parent().removeClass('collapsed'); $(this).parent().addClass('expanded'); } else { $(this).parent().removeClass('expanded'); $(this).parent().addClass('collapsed'); } }); // Reverse order of months. $('.BlogArchive .intervalToggle + div').each(function(_, items) { var year = $(this); year.children().each(function(_, month) { year.prepend(month); }); }); // Set anchors to open in new tab. $('.post-content img').parent().each(function(_, node) { if (node.nodeName == 'A') { $(this).attr('target', '_blank'); } }); // Process search requests. $('.searchBox input').on("keypress", function(ev) { if (ev.which == 13) { window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val()); } }); }); (function($, window) { var archiveButton = $($('#sidebar .widget.BlogArchive h2')[0]); var folderIcon = $('#sidebar .widget.BlogArchive h2::after'); var archivePanel = $('#BlogArchive1'); archiveButton.click(function(e) { if (archivePanel.hasClass('archive-open')) { // It's open, so now we close it archivePanel.removeClass('archive-open'); archivePanel.addClass('archive-closed'); archiveButton.removeClass('archive-open'); archiveButton.addClass('archive-closed'); archivePanel.css('height', '60px'); } else { // It's closed, so open it archivePanel.removeClass('archive-closed'); archivePanel.addClass('archive-open'); archiveButton.removeClass('archive-closed'); archiveButton.addClass('archive-open'); archivePanel.css('height', 'auto'); folderIcon.css('content', 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEggkDzi8QVSua_XxLOnfxWE_8nQc1MAhRPBxJej91JW2ZBbKvb2TjaNf66ZoTqCrBivwRr-TLJYxFm2ZAiMd-zfbJ0TjblXGvUszWFbdRncNOL7jseiJCL9uQoqN8BM8FpesdSw/s1600/keyboard_arrow_up_grey600_24dp.png'); } }); })($, window); (function ($, window, devsite) { devsite.devsite = {}; devsite.expandLeftNav = devsite.expandLeftNav || {}; devsite.expandLeftNav.ANALYTICS_LABEL_ = 'Hamburger menu'; devsite.expandLeftNav.ANALYTICS_ACTION_OPEN_ = 'Open'; devsite.expandLeftNav.ANALYTICS_ACTION_CLOSE_ = 'Close'; devsite.expandLeftNav.lastState_ = 0; devsite.expandLeftNav.chekovEnabled_ = false; devsite.expandLeftNav.init = function () { devsite.expandLeftNav.chekovEnabled_ = true; var navElement; if (devsite.expandLeftNav.chekovEnabled_) { navElement = $('.devsite-nav-responsive'); } else { navElement = $('.devsite-section-nav-responsive'); } this.drawerWidth = parseInt(navElement.css('width')); navElement.css({ 'left': -(this.drawerWidth) }); var expandButton = $('.devsite-expand-section-nav'); expandButton.click(function () { devsite.expandLeftNav.handleNavigationOpened(); }); navElement.find('.devsite-nav-responsive-forward').click( devsite.expandLeftNav.openPanel); navElement.find('.devsite-nav-responsive-back').click( devsite.expandLeftNav.closePanel); if ($('.devsite-nav-responsive-tabs-panel + ' + '.devsite-nav-responsive-sidebar-panel').length) { devsite.expandLeftNav.openPanel(); } }; devsite.expandLeftNav.handleNavigationOpened = function (opt_noAnimate) { var mask; if (devsite.expandLeftNav.chekovEnabled_) { var nav = $('.devsite-nav-responsive'); if (opt_noAnimate) { nav.addClass('devsite-nav-responsive-no-animate'); mask = devsite.devsite.showSiteMask(0); } else { mask = devsite.devsite.showSiteMask(); } nav.addClass('devsite-nav-responsive-open'); } else { devsite.expandLeftNav.lastState_ = devsite.sticky.currentState; devsite.sticky.setState(devsite.sticky.state.COLLAPSED_HEADER); var headerHeight = devsite.sticky.getHeaderHeight() - devsite.sticky.desiredMargin; var drawerHeight = $(window).height() - headerHeight; mask = devsite.devsite.showArticleMask(); $('.devsite-section-nav-responsive').css({ 'height': drawerHeight, 'left': '0', 'top': headerHeight, 'visibility': 'visible' }).focus(); } $(mask).click(devsite.expandLeftNav.handleNavigationClosed); $(document).on('keydown.escape', function (e) { if (e.keyCode == $.ui.keyCode.ESCAPE) { devsite.expandLeftNav.handleNavigationClosed(); $(document).off('keydown.escape'); } }); }; devsite.expandLeftNav.getScrollTop = function () { return $(window).scrollTop(); }; devsite.expandLeftNav.handleNavigationClosed = function () { var nav; if (devsite.expandLeftNav.chekovEnabled_) { nav = $('.devsite-nav-responsive'); devsite.devsite.hideSiteMask(); nav.removeClass('devsite-nav-responsive-open ' + 'devsite-nav-responsive-no-animate'); } else { nav = $('.devsite-section-nav-responsive'); devsite.devsite.hideArticleMask(); devsite.sticky.setState(devsite.expandLeftNav.lastState_); nav .css('left', -(devsite.expandLeftNav.drawerWidth)) .one('transitionend', function () { nav.css('visibility', 'hidden'); }); } nav.scrollTop(0); }; devsite.expandLeftNav.openPanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel.show(); parentPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel .removeClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); parentPanel.hide(); }); setTimeout(function () { childPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); }, 1); }; devsite.expandLeftNav.closePanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); parentPanel .show() .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); setTimeout(function () { parentPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); childPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel.hide(); parentPanel.removeClass('devsite-nav-responsive-transition'); }); }, 1); }; devsite.expandLeftNav.FADE_SLOW_ = 'slow'; devsite.expandLeftNav.FADE_FAST_ = 'fast'; devsite.expandLeftNav.SITE_MASK_CSS_ = '.devsite-site-mask'; devsite.devsite.showSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_SLOW_; } devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, false, opt_animate); }; devsite.devsite.hideSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_FAST_; } devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, true, opt_animate); }; devsite.devsite.showArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_('.devsite-article-mask', false, devsite.expandLeftNav.FADE_SLOW_); }; devsite.devsite.hideArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_('.devsite-article-mask', true, devsite.expandLeftNav.FADE_FAST_); }; devsite.devsite.setMask_ = function(className, out, opt_fadeTime) { var query = $(className); if (opt_fadeTime === 0) { out ? query.hide() : query.show(); } else { out ? query.fadeOut(opt_fadeTime) : query.fadeIn(opt_fadeTime); } return $(className)[0]; }; devsite.devsite.setMouseScrollingEnabled = function(trueOrFalse) { if (trueOrFalse == true) { $('html, body').css({ 'overflow': '' }); } else { $('html, body').css({ 'overflow': 'hidden' }); } }; })($, window, devsite = {}); if (window.jQuery) { $(document).ready(function () { if (window.devsite) { devsite.expandLeftNav.init(); } }); } //]]> </script> <style> .widget ul{ line-height: 1.6 !important; } #sidebar ul li a{ color:black; line-height: 20px; } #sidebar ul li a:hover{ color:#4184F3; } </style> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY5gbKH29HumxIimpgYUhQ7xCLcqGQ:1732467494989';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d15045980','//testing.googleblog.com/search/label/Test%20Flakiness','15045980'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '15045980', 'title': 'Google Testing Blog', 'url': 'https://testing.googleblog.com/search/label/Test%20Flakiness', 'canonicalUrl': 'https://testing.googleblog.com/search/label/Test%20Flakiness', 'homepageUrl': 'https://testing.googleblog.com/', 'searchUrl': 'https://testing.googleblog.com/search', 'canonicalHomepageUrl': 'https://testing.googleblog.com/', 'blogspotFaviconUrl': 'https://testing.googleblog.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'G-838ZCPQWM6', 'analytics4': true, 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Google Testing Blog - RSS\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/15045980/posts/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/d78375fb222d99b3', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'index', 'searchLabel': 'Test Flakiness', 'pageName': 'Test Flakiness', 'pageTitle': 'Google Testing Blog: Test Flakiness'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Google Testing Blog', 'description': '', 'url': 'https://testing.googleblog.com/search/label/Test%20Flakiness', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': false, 'isSearch': true, 'isLabelSearch': true, 'search': {'label': 'Test Flakiness', 'resultsMessage': 'Showing posts with the label Test Flakiness', 'resultsMessageHtml': 'Showing posts with the label \x3cspan class\x3d\x27search-label\x27\x3eTest Flakiness\x3c/span\x3e'}}}]); _WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'navMessage': 'Showing posts with label \x3cb\x3eTest Flakiness\x3c/b\x3e. \x3ca href\x3d\x22https://testing.googleblog.com/\x22\x3eShow all posts\x3c/a\x3e', 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/2646514562-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML9', 'sidebar-bottom', document.getElementById('HTML9'), {}, 'displayModeFull')); </script> </body> </html>