CINXE.COM

<!DOCTYPE html> <html class='v2 list-page' dir='ltr' itemscope='' itemtype='http://schema.org/Blog' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <title> Google Testing Blog: 2015 </title> <meta content='width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0' name='viewport'/> <meta content='IE=Edge' http-equiv='X-UA-Compatible'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='en_US' property='og:locale'/> <meta content='https://testing.googleblog.com/2015/' property='og:url'/> <meta content='Google Testing Blog' property='og:site_name'/>  <meta content='Google Testing Blog' property='og:title'/> <meta content='summary' name='twitter:card'/> <meta content='@googletesting' name='twitter:creator'/> <link href='https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic' rel='stylesheet' type='text/css'/> <link href='https://fonts.googleapis.com/icon?family=Material+Icons' rel='stylesheet'/> <script src='https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js' type='text/javascript'></script>  <style id='page-skin-1' type='text/css'></style> <style id='template-skin-1' type='text/css'></style>  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://testing.googleblog.com/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://testing.googleblog.com/2015/' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://testing.googleblog.com/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="Google Testing Blog - RSS" href="https://testing.googleblog.com/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://www.blogger.com/feeds/15045980/posts/default" />  <meta content='https://testing.googleblog.com/2015/' property='og:url'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='' property='og:description'/>  <base target='_self'/> <style> html { font-family: Roboto, sans-serif; -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; } body { padding: 0; /* This ensures that the scroll bar is always present, which is needed */ /* because content render happens after page load; otherwise the header */ /* would "bounce" in-between states. */ min-height: 150%; } h2 { font-size: 20px; } h1, h2, h3, h4, h5 { line-height: 2em; } html, h4, h5, h6 { font-size: 17px; } h3 { font-size: 18px !important; } a, a:visited { color: #4184F3; text-decoration: none; } a:focus, a:hover, a:active { text-decoration: none; } .Header { margin-top: 15px; } .Header h1 { font-size: 32px; font-weight: 300; line-height: 32px; height: 42px; } .header-inner .Header .titlewrapper { padding: 0; margin-top: 30px; } .header-inner .Header .descriptionwrapper { padding: 0; margin: 0; } .cols-wrapper { margin-top: 56px; } .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 60px; } .header-inner { height: 256px; position: relative; } html, .header-inner a { color: #212121; color: rgba(0,0,0,.87); } .header-inner .google-logo { display: inline-block; background-size: contain; z-index: 1; height: 46px; overflow: hidden; margin-top: 4px; margin-right: 8px; } .header-left { position: absolute; top: 50%; -webkit-transform: translateY(-50%); transform: translateY(-50%); margin-top: 12px; width: 100%; } .google-logo { margin-left: -4px; } #google-footer { position: relative; font-size: 13px; list-style: none; text-align: right; } #google-footer a { color: #444; } #google-footer ul { margin: 0; padding: 0; height: 144px; line-height: 144px; } #google-footer ul li { display: inline; } #google-footer ul li:before { color: #999; content: "\00b7"; font-weight: bold; margin: 5px; } #google-footer ul li:first-child:before { content: ''; } #google-footer .google-logo-dark { left: 0; margin-top: -16px; position: absolute; top: 50%; } /** Sitemap links. **/ .footer-inner-2 { font-size: 14px; padding-top: 42px; padding-bottom: 74px; } .footer-inner-2 .HTML h2 { color: #212121; color: rgba(0,0,0,.87); font-size: 14px; font-weight: 500; padding-left: 0; margin: 10px 0; } .footer-inner-2 .HTML ul { font-weight: normal; list-style: none; padding-left: 0; } .footer-inner-2 .HTML li { line-height: 24px; padding: 0; } .footer-inner-2 li a { color: rgba(65,132,243,.87); } /** Archive widget. **/ .BlogArchive { font-size: 13px; font-weight: normal; } .BlogArchive .widget-content { display: none; } .BlogArchive h2, .Label h2 { color: #4184F3; text-decoration: none; } .BlogArchive .hierarchy li { display: inline-block; } /* Specificity needed here to override widget CSS defaults. */ .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li { padding-left: 0; text-indent: 0; } .BlogArchive .intervalToggle { cursor: pointer; } .BlogArchive .expanded .intervalToggle .new-toggle { -ms-transform: rotate(180deg); transform: rotate(180deg); } .BlogArchive .new-toggle { float: right; padding-top: 3px; opacity: 0.87; } #ArchiveList { text-transform: uppercase; } #ArchiveList .expanded > ul:last-child { margin-bottom: 16px; } #ArchiveList .archivedate { width: 100%; } /* Months */ .BlogArchive .items { max-width: 150px; margin-left: -4px; } .BlogArchive .expanded .items { margin-bottom: 10px; overflow: hidden; } .BlogArchive .items > ul { float: left; height: 32px; } .BlogArchive .items a { padding: 0 4px; } .Label { font-size: 13px; font-weight: normal; } .sidebar-icon { display: inline-block; width: 24px; height: 24px; vertical-align: middle; margin-right: 12px; margin-top: -1px } .Label a { margin-right: 4px; } .Label .widget-content { display: none; } .FollowByEmail { font-size: 13px; font-weight: normal; } .FollowByEmail h2 { background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); background-repeat: no-repeat; background-position: 0 50%; text-indent: 30px; } .FollowByEmail .widget-content { display: none; } .searchBox input { border: 1px solid #eee; color: #212121; color: rgba(0,0,0,.87); font-size: 14px; padding: 8px 8px 8px 40px; width: 164px; font-family: Roboto, sans-serif; background: url("https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png") 8px center no-repeat; } .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */ color: rgba(0,0,0,.54); } .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */ color: #000; opacity: 0.54; } .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */ color: #000; opacity: 0.54; } .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */ color: #757575; } .widget-item-control { margin-top: 0px; } .section { margin: 0; padding: 0; } #sidebar-top { border: 1px solid #eee; } #sidebar-top > div { margin: 16px 0; } .widget ul { line-height: 1.6; } /*main post*/ .post { margin-bottom:30px; } #main .post .title { margin: 0; } #main .post .title a { color: #212121; color: rgba(0,0,0,.87); font-weight: normal; font-size: 24px; } #main .post .title a:hover { text-decoration:none; color:#4184F3; } .message, #main .post .post-header { margin: 0; padding: 0; } #main .post .post-header .caption, #main .post .post-header .labels-caption, #main .post .post-footer .caption, #main .post .post-footer .labels-caption { color: #444; font-weight: 500; } #main .tr-caption-container td { text-align: left; } #main .post .tr-caption { color: #757575; color: rgba(0,0,0,.54); display: block; max-width: 560px; padding-bottom: 20px; } #main .post .tr-caption-container { line-height: 24px; margin: -1px 0 0 0 !important; padding: 4px 0; text-align: left; } #main .post .post-header .published{ font-size:11px; font-weight:bold; } .post-header .publishdate { font-size: 17px; font-weight:normal; color: #757575; color: rgba(0,0,0,.54); } #main .post .post-footer{ font-size:12px; padding-bottom: 21px; } .label-footer { margin-bottom: 12px; margin-top: 12px; } .comment-img { margin-right: 16px; opacity: 0.54; vertical-align: middle; } #main .post .post-header .published { margin-bottom: 40px; margin-top: -2px; } .post .post-content { color: #212121; color: rgba(0,0,0,.87); font-size: 17px; margin: 25px 0 36px 0; line-height: 32px; word-wrap: break-word; } .post-body .post-content ul, .post-body .post-content ol { margin: 16px 0; padding: 0 48px; } .post-summary { display: none; } /* Capture section headers. */ .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b { display: inline-block; margin-bottom: 8px; margin-top: 24px; } .post-content li { line-height: 32px; } /* Override all post images/videos to left align. */ .post-content .separator, .post-content > div { text-align: left; } .post-content .separator > a, .post-content .separator > span { margin-left: 0 !important; } .post-content img { max-width: 100%; } .post-content .tr-caption-container img { margin-bottom: 12px; } .post-content iframe, .post-content embed { max-width: 100%; } .post-content .carousel-container { margin-bottom: 48px; } #main .post-content b { font-weight: 500; } /* These are the main paragraph spacing tweaks. */ #main .post-content br { content: ' '; display: block; padding: 4px; } .post-content .space { display: block; height: 8px; } .post-content iframe + .space, .post-content iframe + br { padding: 0 !important; } #main .post .jump-link { margin-bottom:10px; } .post-content img, .post-content iframe { margin: 30px 0 20px 0; } .post-content > img:first-child, .post-content > iframe:first-child { margin-top: 0; } .col-right .section { padding: 0 16px; } #aside { background:#fff; border:1px solid #eee; border-top: 0; } #aside .widget { margin:0; } #aside .widget h2, #ArchiveList .toggle + a.post-count-link { color: #212121; color: rgba(0,0,0,.87); font-weight: 400 !important; margin: 0; } #ArchiveList .toggle { float: right; } #ArchiveList .toggle .material-icons { padding-top: 4px; } #sidebar .tab { cursor: pointer; } #sidebar .tab .arrow { display: inline-block; float: right; } #sidebar .tab .icon { display: inline-block; vertical-align: top; height: 24px; width: 24px; margin-right: 13px; margin-left: -1px; margin-top: 1px; color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget-content > :first-child { padding-top: 8px; } #sidebar .active .tab .arrow { -ms-transform: rotate(180deg); transform: rotate(180deg); } #sidebar .arrow { color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget h2 { font-size: 14px; line-height: 24px; display: inline-block; } #sidebar .widget .BlogArchive { padding-bottom: 8px; } #sidebar .widget { border-bottom: 1px solid #eee; box-shadow: 0px 1px 0 white; margin-bottom: 0; padding: 14px 0; min-height: 20px; } #sidebar .widget:last-child { border-bottom: none; box-shadow: none; margin-bottom: 0; } #sidebar ul { margin: 0; padding: 0; } #sidebar ul li { list-style:none; padding:0; } #sidebar ul li a { line-height: 32px; } #sidebar .archive { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC"); height: 24px; line-height: 24px; padding-left: 30px; } #sidebar .labels { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC"); height: 20px; line-height: 20px; padding-left: 30px; } #sidebar .rss a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII="); } #sidebar .subscription a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); } #sidebar-bottom { background: #f5f5f5; border-top:1px solid #eee; } #sidebar-bottom .widget { border-bottom: 1px solid #e0e0e0; padding: 15px 0; text-align: center; } #sidebar-bottom > div:last-child { border-bottom: 0; } #sidebar-bottom .text { line-height: 20px; } /* Home, forward, and backward pagination. */ .blog-pager { border-top : 1px #e0e0e0 solid; padding-top: 10px; margin-top: 15px; text-align: right !important; } #blog-pager { margin-botom: 0; margin-top: -14px; padding: 16px 0 0 0; } #blog-pager a { display: inline-block; } .blog-pager i.disabled { opacity: 0.2 !important; } .blog-pager i { color: black; margin-left: 16px; opacity: 0.54; } .blog-pager i:hover, .blog-pager i:active { opacity: 0.87; } #blog-pager-older-link, #blog-pager-newer-link { float: none; } .gplus-profile { background-color: #fafafa; border: 1px solid #eee; overflow: hidden; width: 212px; } .gplus-profile-inner { margin-left: -1px; margin-top: -1px; } /* Sidebar follow buttons. */ .followgooglewrapper { padding: 12px 0 0 0; } .loading { visibility: hidden; } .detail-page .post-footer .cmt_iframe_holder { padding-top: 40px !important; } /** Desktop **/ @media (max-width: 900px) { .col-right { display: none; } .col-main { margin-right: 0; min-width: initial; } .footer-outer { display: none; } .cols-wrapper { min-width: initial; } .google-footer-outer { background-color: #f5f5f5; } } /** Tablet **/ @media (max-width: 712px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 40px; } } /* An extra breakpoint accommodating for long blog titles. */ @media (max-width: 600px) { .header-left { height: 100%; position: initial; top: inherit; margin-top: 0; -webkit-transform: initial; transform: initial; } .header-title { margin-top: 18px; } .header-inner { height: auto; margin-bottom: 32px; margin-top: 32px; } .header-desc { margin-top: 12px; } .header-inner .google-logo { height: 40px; margin-top: 3px; } .header-inner .google-logo img { height: 42px; } .header-title h2 { font-size: 32px; line-height: 40px; } } /** Mobile/small desktop window; also landscape. **/ @media (max-width: 480px), (max-height: 480px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 16px; } .cols-wrapper { margin-top: 0; } .post-header .publishdate, .post .post-content { font-size: 16px; } .post .post-content { line-height: 28px; margin-bottom: 30px; } .post { margin-top: 30px; } .byline-author { display: block; font-size: 12px; line-height: 24px; margin-top: 6px; } #main .post .title a { font-weight: 500; color: #4c4c4c; color: rgba(0,0,0,.70); } #main .post .post-header { padding-bottom: 12px; } #main .post .post-header .published { margin-bottom: -8px; margin-top: 3px; } .post .read-more { display: block; margin-top: 14px; } .post .tr-caption { font-size: 12px; } #main .post .title a { font-size: 20px; line-height: 30px; } .post-content iframe { /* iframe won't keep aspect ratio when scaled down. */ max-height: 240px; } .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe { margin-left: -16px; max-width: inherit; width: calc(100% + 32px); } .post-content table, .post-content td { width: 100%; } #blog-pager { margin: 0; padding: 16px 0; } /** List page tweaks. **/ .list-page .post-original { display: none; } .list-page .post-summary { display: block; } .list-page .comment-container { display: none; } .list-page #blog-pager { padding-top: 0; border: 0; margin-top: -8px; } .list-page .label-footer { display: none; } .list-page #main .post .post-footer { border-bottom: 1px solid #eee; margin: -16px 0 0 0; padding: 0 0 20px 0; } .list-page .post .share { display: none; } /** Detail page tweaks. **/ .detail-page .post-footer .cmt_iframe_holder { padding-top: 32px !important; } .detail-page .label-footer { margin-bottom: 0; } .detail-page #main .post .post-footer { padding-bottom: 0; } .detail-page #comments { display: none; } } [data-about-pullquote], [data-is-preview], [data-about-syndication] { display: none; } </style> <noscript> <style> .loading { visibility: visible }</style> </noscript>  <script async='true' src='https://www.googletagmanager.com/gtag/js?id=G-838ZCPQWM6'></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-838ZCPQWM6'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=29523768-5e4c-476e-8247-8f020a274c86' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=29523768-5e4c-476e-8247-8f020a274c86' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <script type='text/javascript'> //<![CDATA[ var axel = Math.random() + ""; var a = axel * 10000000000000; document.write('<iframe src="https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?" width="1" height="1" frameborder="0" style="display:none"></iframe>'); //]]> </script> <noscript> <img alt='' height='1' src='https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?' width='1'/> </noscript>  <div class='header-outer'> <div class='header-inner'> <div class='section' id='header'><div class='widget Header' data-version='1' id='Header1'> <div class='header-left'> <div class='header-title'> <a class='google-logo' href='https://testing.googleblog.com/'> <img height='50' src='https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png'/> </a> <a href='/.'> <h2> Testing Blog </h2> </a> </div> <div class='header-desc'> </div> </div> </div></div> </div> </div>  <div class='cols-wrapper loading'> <div class='col-main-wrapper'> <div class='col-main'> <div class='section' id='main'><div class='widget Blog' data-version='1' id='Blog1'> <div class='post' data-id='7218196899874976601' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/12/gtac-2015-wrap-up.html' itemprop='url' title='GTAC 2015 Wrap Up'> GTAC 2015 Wrap Up </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, December 08, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by Michael Klepikov and Lesley Katzen on behalf of the GTAC Committee</i><br /> <br /> The ninth <a href="//g.co/gtac">GTAC</a> (Google Test Automation Conference) was held on November 10-11 at the <a href="//www.google.com/about/careers/locations/cambridge/">Google Cambridge</a> office, the “Hub” of innovation. The conference was completely packed with presenters and attendees from all over the world, from industry and academia, discussing advances in test automation and the test engineering computer science field, bringing with them a huge diversity of experiences. Speakers from numerous companies and universities (Applitools, Automattic, Bitbar, Georgia Tech, Google, Indian Institute of Science, Intel, LinkedIn, Lockheed Martin, MIT, Nest, Netflix, OptoFidelity, Splunk, Supersonic, Twitter, Uber, University of Waterloo) spoke on a variety of interesting and cutting edge test automation topics. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5TtkE1lv1VVllCMtVKUZiAZjSOkBBwPjW0KLnRrtHvGkqW4A7klIU7jt4Ik6ozu7tdT3FV2NEfS94yVTlAhmJngx9KKRFgDJ8V3MxEUxX4_KnqmOffBitwFmp54MGVYoz9i1_/s1600/GTAC2015-4520.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5TtkE1lv1VVllCMtVKUZiAZjSOkBBwPjW0KLnRrtHvGkqW4A7klIU7jt4Ik6ozu7tdT3FV2NEfS94yVTlAhmJngx9KKRFgDJ8V3MxEUxX4_KnqmOffBitwFmp54MGVYoz9i1_/s400/GTAC2015-4520.jpg" width="400" /></a></div> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiZc_NFxSEyeBtaNPxwzQ43MhZrai3C04BrtihWiSuU9PlTaGKgT5IZ6YAul88vn3BS-pmNbuxs3WfqOTBG4KEVNpMD8P4nrN-oC2YULPKqjpFPdQqDGbSKyGUabwxD3_1uxXsj/s1600/GTAC2015-4964.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiZc_NFxSEyeBtaNPxwzQ43MhZrai3C04BrtihWiSuU9PlTaGKgT5IZ6YAul88vn3BS-pmNbuxs3WfqOTBG4KEVNpMD8P4nrN-oC2YULPKqjpFPdQqDGbSKyGUabwxD3_1uxXsj/s400/GTAC2015-4964.jpg" width="400" /></a></div> <br /> All presentation videos and slides are posted on the <a href="https://developers.google.com/google-test-automation-conference/2015/stream">Video Recordings</a> and <a href="https://developers.google.com/google-test-automation-conference/2015/presentations">Presentations</a> pages. All videos have professionally transcribed closed captions, and the YouTube descriptions have the slides links. Enjoy and share! <br /> <br /> We had over 1,300 applicants and over 200 of those for speaking. Over 250 people filled our venue to capacity, and the live stream had a peak of about 400 concurrent viewers, with about 3,300 total viewing hours. <br /> <br /> Our goal in hosting GTAC is to make the conference highly relevant and useful for both attendees and the larger test engineering community as a whole. Our post-conference survey shows that we are close to achieving that goal; thanks to everyone who completed the feedback survey!<br /> <br /> <ul> <li>Our 82 survey respondents were mostly (81%) test focused professionals with a wide range of 1 to 40 years of experience. </li> <li>Another 76% of respondents rated the conference as a whole as above average, with marked satisfaction for the venue, the food (those Diwali treats!), and the breadth and coverage of the talks themselves.</li> </ul> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjONDspgit8yszimfaKcT70z2SUeVSSChO-ztQHa6V1is0fPEcW21o4gq45y5FkfVlYGWC6vWGfdLDckT2rtmNDG_Yh50vmE8P2fYy_dJ4hZzbN1Wa5Is-JRs75u0S5JYq3hkfC/s1600/GTAC2015-4819.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjONDspgit8yszimfaKcT70z2SUeVSSChO-ztQHa6V1is0fPEcW21o4gq45y5FkfVlYGWC6vWGfdLDckT2rtmNDG_Yh50vmE8P2fYy_dJ4hZzbN1Wa5Is-JRs75u0S5JYq3hkfC/s400/GTAC2015-4819.jpg" width="400" /></a></div> <br /> The top five most popular talks were:<br /> <br /> <ul> <li>The Uber Challenge of Cross-Application/Cross-Device Testing (Apple Chow and Bian Jiang) </li> <li>Your Tests Aren't Flaky (Alister Scott) </li> <li>Statistical Data Sampling (Celal Ziftci and Ben Greenberg) </li> <li>Coverage is Not Strongly Correlated with Test Suite Effectiveness (Laura Inozemtseva) </li> <li>Chrome OS Test Automation Lab (Simran Basi and Chris Sosa).</li> </ul> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiGffKTqtYqbA5a7RuLC0aXwP1EwGuRbFY4n68nBI5ZfUoLryevJb4YBAIMg4Sn1hVTdfMQ3TCi6KN2IpUo4irB4PL-h6qMdvml9DzwQL4WISmhACAiFmuk2ig4LLUE1GTS7BWo/s1600/GTAC2015-4657.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiGffKTqtYqbA5a7RuLC0aXwP1EwGuRbFY4n68nBI5ZfUoLryevJb4YBAIMg4Sn1hVTdfMQ3TCi6KN2IpUo4irB4PL-h6qMdvml9DzwQL4WISmhACAiFmuk2ig4LLUE1GTS7BWo/s400/GTAC2015-4657.jpg" width="266" /></a></div> <br /> Our social events also proved to be crowd pleasers. The social events were a direct response to feedback from GTAC 2014 for organized opportunities for socialization among the GTAC attendees. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEht3Sc6wM5IwvjSKcXzr3MhH4Xec2TAE2CDo1bQh9ziYHLmLq1yZxnTrH9XsW3Rp5eSEsqJmU6yKE4izayEveixIool5zevYPcAx9gyI6y2qYj_kDAtpvAUpQi-8LERSGaGgFnv/s1600/GTAC2015-4871.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEht3Sc6wM5IwvjSKcXzr3MhH4Xec2TAE2CDo1bQh9ziYHLmLq1yZxnTrH9XsW3Rp5eSEsqJmU6yKE4izayEveixIool5zevYPcAx9gyI6y2qYj_kDAtpvAUpQi-8LERSGaGgFnv/s400/GTAC2015-4871.jpg" width="400" /></a></div> <br /> This isn’t to say there isn’t room for improvement. We had 11% of respondents express frustration with event communications and provided some long, thoughtful suggestions for what we could do to improve next year. Also, many of the long form comments asked for a better mix of technologies, noting that mobile had a big presence in the talks this year. <br /> <br /> If you have any suggestions on how we can improve, please comment on this post, or better yet – fill out the <a href="https://docs.google.com/a/google.com/forms/d/1_1Jb23jsTxoVat-HGoKJyJ9BcfdvoDWIpatLqv6fTNA/viewform">survey</a>, which remains open. Based on feedback from last year urging more transparency in speaker selection, we included an individual outside of Google in the speaker evaluation. Feedback is precious, we take it very seriously, and we will use it to improve next time around. <br /> <br /> Thank you to all the speakers, attendees, and online viewers who made this a special event once again. To receive announcements about the next GTAC, currently planned for early 2017, subscribe to the <a href="http://googletesting.blogspot.com/">Google Testing Blog</a>. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by Michael Klepikov and Lesley Katzen on behalf of the GTAC Committee</i><br /> <br /> The ninth <a href="//g.co/gtac">GTAC</a> (Google Test Automation Conference) was held on November 10-11 at the <a href="//www.google.com/about/careers/locations/cambridge/">Google Cambridge</a> office, the “Hub” of innovation. The conference was completely packed with presenters and attendees from all over the world, from industry and academia, discussing advances in test automation and the test engineering computer science field, bringing with them a huge diversity of experiences. Speakers from numerous companies and universities (Applitools, Automattic, Bitbar, Georgia Tech, Google, Indian Institute of Science, Intel, LinkedIn, Lockheed Martin, MIT, Nest, Netflix, OptoFidelity, Splunk, Supersonic, Twitter, Uber, University of Waterloo) spoke on a variety of interesting and cutting edge test automation topics. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5TtkE1lv1VVllCMtVKUZiAZjSOkBBwPjW0KLnRrtHvGkqW4A7klIU7jt4Ik6ozu7tdT3FV2NEfS94yVTlAhmJngx9KKRFgDJ8V3MxEUxX4_KnqmOffBitwFmp54MGVYoz9i1_/s1600/GTAC2015-4520.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5TtkE1lv1VVllCMtVKUZiAZjSOkBBwPjW0KLnRrtHvGkqW4A7klIU7jt4Ik6ozu7tdT3FV2NEfS94yVTlAhmJngx9KKRFgDJ8V3MxEUxX4_KnqmOffBitwFmp54MGVYoz9i1_/s400/GTAC2015-4520.jpg" width="400" /></a></div> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiZc_NFxSEyeBtaNPxwzQ43MhZrai3C04BrtihWiSuU9PlTaGKgT5IZ6YAul88vn3BS-pmNbuxs3WfqOTBG4KEVNpMD8P4nrN-oC2YULPKqjpFPdQqDGbSKyGUabwxD3_1uxXsj/s1600/GTAC2015-4964.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiZc_NFxSEyeBtaNPxwzQ43MhZrai3C04BrtihWiSuU9PlTaGKgT5IZ6YAul88vn3BS-pmNbuxs3WfqOTBG4KEVNpMD8P4nrN-oC2YULPKqjpFPdQqDGbSKyGUabwxD3_1uxXsj/s400/GTAC2015-4964.jpg" width="400" /></a></div> <br /> All presentation videos and slides are posted on the <a href="https://developers.google.com/google-test-automation-conference/2015/stream">Video Recordings</a> and <a href="https://developers.google.com/google-test-automation-conference/2015/presentations">Presentations</a> pages. All videos have professionally transcribed closed captions, and the YouTube descriptions have the slides links. Enjoy and share! <br /> <br /> We had over 1,300 applicants and over 200 of those for speaking. Over 250 people filled our venue to capacity, and the live stream had a peak of about 400 concurrent viewers, with about 3,300 total viewing hours. <br /> <br /> Our goal in hosting GTAC is to make the conference highly relevant and useful for both attendees and the larger test engineering community as a whole. Our post-conference survey shows that we are close to achieving that goal; thanks to everyone who completed the feedback survey!<br /> <br /> <ul> <li>Our 82 survey respondents were mostly (81%) test focused professionals with a wide range of 1 to 40 years of experience. </li> <li>Another 76% of respondents rated the conference as a whole as above average, with marked satisfaction for the venue, the food (those Diwali treats!), and the breadth and coverage of the talks themselves.</li> </ul> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjONDspgit8yszimfaKcT70z2SUeVSSChO-ztQHa6V1is0fPEcW21o4gq45y5FkfVlYGWC6vWGfdLDckT2rtmNDG_Yh50vmE8P2fYy_dJ4hZzbN1Wa5Is-JRs75u0S5JYq3hkfC/s1600/GTAC2015-4819.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjONDspgit8yszimfaKcT70z2SUeVSSChO-ztQHa6V1is0fPEcW21o4gq45y5FkfVlYGWC6vWGfdLDckT2rtmNDG_Yh50vmE8P2fYy_dJ4hZzbN1Wa5Is-JRs75u0S5JYq3hkfC/s400/GTAC2015-4819.jpg" width="400" /></a></div> <br /> The top five most popular talks were:<br /> <br /> <ul> <li>The Uber Challenge of Cross-Application/Cross-Device Testing (Apple Chow and Bian Jiang) </li> <li>Your Tests Aren't Flaky (Alister Scott) </li> <li>Statistical Data Sampling (Celal Ziftci and Ben Greenberg) </li> <li>Coverage is Not Strongly Correlated with Test Suite Effectiveness (Laura Inozemtseva) </li> <li>Chrome OS Test Automation Lab (Simran Basi and Chris Sosa).</li> </ul> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiGffKTqtYqbA5a7RuLC0aXwP1EwGuRbFY4n68nBI5ZfUoLryevJb4YBAIMg4Sn1hVTdfMQ3TCi6KN2IpUo4irB4PL-h6qMdvml9DzwQL4WISmhACAiFmuk2ig4LLUE1GTS7BWo/s1600/GTAC2015-4657.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiGffKTqtYqbA5a7RuLC0aXwP1EwGuRbFY4n68nBI5ZfUoLryevJb4YBAIMg4Sn1hVTdfMQ3TCi6KN2IpUo4irB4PL-h6qMdvml9DzwQL4WISmhACAiFmuk2ig4LLUE1GTS7BWo/s400/GTAC2015-4657.jpg" width="266" /></a></div> <br /> Our social events also proved to be crowd pleasers. The social events were a direct response to feedback from GTAC 2014 for organized opportunities for socialization among the GTAC attendees. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEht3Sc6wM5IwvjSKcXzr3MhH4Xec2TAE2CDo1bQh9ziYHLmLq1yZxnTrH9XsW3Rp5eSEsqJmU6yKE4izayEveixIool5zevYPcAx9gyI6y2qYj_kDAtpvAUpQi-8LERSGaGgFnv/s1600/GTAC2015-4871.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEht3Sc6wM5IwvjSKcXzr3MhH4Xec2TAE2CDo1bQh9ziYHLmLq1yZxnTrH9XsW3Rp5eSEsqJmU6yKE4izayEveixIool5zevYPcAx9gyI6y2qYj_kDAtpvAUpQi-8LERSGaGgFnv/s400/GTAC2015-4871.jpg" width="400" /></a></div> <br /> This isn’t to say there isn’t room for improvement. We had 11% of respondents express frustration with event communications and provided some long, thoughtful suggestions for what we could do to improve next year. Also, many of the long form comments asked for a better mix of technologies, noting that mobile had a big presence in the talks this year. <br /> <br /> If you have any suggestions on how we can improve, please comment on this post, or better yet – fill out the <a href="https://docs.google.com/a/google.com/forms/d/1_1Jb23jsTxoVat-HGoKJyJ9BcfdvoDWIpatLqv6fTNA/viewform">survey</a>, which remains open. Based on feedback from last year urging more transparency in speaker selection, we included an individual outside of Google in the speaker evaluation. Feedback is precious, we take it very seriously, and we will use it to improve next time around. <br /> <br /> Thank you to all the speakers, attendees, and online viewers who made this a special event once again. To receive announcements about the next GTAC, currently planned for early 2017, subscribe to the <a href="http://googletesting.blogspot.com/">Google Testing Blog</a>. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2015 Wrap Up&url=https://testing.googleblog.com/2015/12/gtac-2015-wrap-up.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/12/gtac-2015-wrap-up.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/12/gtac-2015-wrap-up.html#comments' style='font-weight: 500; text-decoration: underline;'>10 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/12/gtac-2015-wrap-up.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> , <a class='label' href='https://testing.googleblog.com/search/label/Lesley%20Katzen' rel='tag'> Lesley Katzen </a> , <a class='label' href='https://testing.googleblog.com/search/label/Michael%20Klepikov' rel='tag'> Michael Klepikov </a> </span> </div> </div> </div> <div class='post' data-id='9146747531209300966' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/11/gtac-2015-is-next-week.html' itemprop='url' title='GTAC 2015 is Next Week!'> GTAC 2015 is Next Week! </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Friday, November 06, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee</i><br /> <br /> The ninth GTAC (Google Test Automation Conference) commences on Tuesday, November 10th, at the Google Cambridge office. You can find the latest details on the <a href="https://developers.google.com/google-test-automation-conference/">conference site</a>, including <a href="https://developers.google.com/google-test-automation-conference/2015/schedule">schedule</a>, <a href="https://developers.google.com/google-test-automation-conference/2015/speakers">speaker profiles</a>, and <a href="https://developers.google.com/google-test-automation-conference/2015/travel">travel tips</a>.<br /> <br /> If you have not been invited to attend in person, you can <a href="https://developers.google.com/google-test-automation-conference/2015/stream">watch the event live</a>. And if you miss the livestream, we will post slides and videos later.<br /> <br /> We have an outstanding speaker lineup this year, and we look forward to seeing you all there or online! <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee</i><br /> <br /> The ninth GTAC (Google Test Automation Conference) commences on Tuesday, November 10th, at the Google Cambridge office. You can find the latest details on the <a href="https://developers.google.com/google-test-automation-conference/">conference site</a>, including <a href="https://developers.google.com/google-test-automation-conference/2015/schedule">schedule</a>, <a href="https://developers.google.com/google-test-automation-conference/2015/speakers">speaker profiles</a>, and <a href="https://developers.google.com/google-test-automation-conference/2015/travel">travel tips</a>.<br /> <br /> If you have not been invited to attend in person, you can <a href="https://developers.google.com/google-test-automation-conference/2015/stream">watch the event live</a>. And if you miss the livestream, we will post slides and videos later.<br /> <br /> We have an outstanding speaker lineup this year, and we look forward to seeing you all there or online! <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2015 is Next Week!&url=https://testing.googleblog.com/2015/11/gtac-2015-is-next-week.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/11/gtac-2015-is-next-week.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/11/gtac-2015-is-next-week.html#comments' style='font-weight: 500; text-decoration: underline;'>3 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/11/gtac-2015-is-next-week.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </span> </div> </div> </div> <div class='post' data-id='953171939960556151' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/10/announcing-gtac-2015-agenda.html' itemprop='url' title='Announcing the GTAC 2015 Agenda'> Announcing the GTAC 2015 Agenda </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, October 27, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> We have completed the selection and confirmation of all speakers and attendees for GTAC 2015. You can find the detailed agenda at: <a href="http://developers.google.com/gtac/2015/schedule">developers.google.com/gtac/2015/schedule</a>.<br /> <br /> Thank you to all who submitted proposals!<br /> <br /> There is a lot of interest in GTAC once again this year with about 1400 applicants and about 200 of those for speaking. Unfortunately, our venue only seats 250. We will livestream the event as usual, so fret not if you were not selected to attend. Information about the livestream and other details will be posted on the GTAC site soon and announced here. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> We have completed the selection and confirmation of all speakers and attendees for GTAC 2015. You can find the detailed agenda at: <a href="http://developers.google.com/gtac/2015/schedule">developers.google.com/gtac/2015/schedule</a>.<br /> <br /> Thank you to all who submitted proposals!<br /> <br /> There is a lot of interest in GTAC once again this year with about 1400 applicants and about 200 of those for speaking. Unfortunately, our venue only seats 250. We will livestream the event as usual, so fret not if you were not selected to attend. Information about the livestream and other details will be posted on the GTAC site soon and announced here. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Announcing the GTAC 2015 Agenda&url=https://testing.googleblog.com/2015/10/announcing-gtac-2015-agenda.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/10/announcing-gtac-2015-agenda.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/10/announcing-gtac-2015-agenda.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/10/announcing-gtac-2015-agenda.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </span> </div> </div> </div> <div class='post' data-id='416454539637738321' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/10/audio-testing-automatic-gain-control.html' itemprop='url' title='Audio Testing - Automatic Gain Control'> Audio Testing - Automatic Gain Control </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Thursday, October 08, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>By: Patrik Höglund </i><br /> <br /> <h2> What is Automatic Gain Control? </h2> It’s time to talk about advanced media quality tests again! As experienced Google testing blog readers know, when I write an article it’s usually about WebRTC, and the unusual testing solutions we build to test it. This article is no exception. Today we’re going to talk about Automatic Gain Control, or AGC. This is a feature that’s on by default for WebRTC applications, such as <a href="http://apprtc.appspot.com/">http://apprtc.appspot.com</a>. It uses various means to adjust the microphone signal so your voice makes it loud and clear to the other side of the peer connection. For instance, it can attempt to adjust your microphone gain or try to amplify the signal digitally.<br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhm7lOY1ZMr0tyR49jxejFz4IsseLaVb67h8akatZjCEs8pIRZFjhH4-980o0YjZ_ZzRuvyPzIfch-V3CgPIBoH8pqrkhAeW6GNRqv6K9uIZkU1EOM6rvbl9BvT5Zo1gMgArCEs/s1600/image03.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="336" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhm7lOY1ZMr0tyR49jxejFz4IsseLaVb67h8akatZjCEs8pIRZFjhH4-980o0YjZ_ZzRuvyPzIfch-V3CgPIBoH8pqrkhAeW6GNRqv6K9uIZkU1EOM6rvbl9BvT5Zo1gMgArCEs/s640/image03.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 1. How Auto Gain Control works [<a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_processing/agc/agc_manager_direct.h&sq=package:chromium">code here</a>]. </i></div> <div style="text-align: center;"> <br /></div> This is an example of automatic control engineering (another example would be the classic <a href="https://en.wikipedia.org/wiki/PID_controller">PID controller</a>) and happens in real time. Therefore, if you move closer to the mic while speaking, the AGC will notice the output stream is too loud, and reduce mic volume and/or digital gain. When you move further away, it tries to adapt up again. The fancy voice activity detector is there so we only amplify speech, and not, say, the microwave oven your spouse just started in the other room.<br /> <h2> Testing the AGC</h2> Now, how do we make sure the AGC works? The first thing is obviously to <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_processing/agc/agc_unittest.cc&sq=package:chromium">write unit tests</a> and <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_processing/test/">integration tests</a>. You <a href="http://googletesting.blogspot.com/2015/04/just-say-no-to-more-end-to-end-tests.html">didn’t think about building that end-to-end test first, did you</a>? Once we have the lower-level tests in place, we can start looking at a bigger test. While developing the WebRTC implementation in Chrome, we had several bugs where the AGC code was working by itself, but was misconfigured in Chrome. In one case, it was simply turned off for all users. In another, it was only turned off in Hangouts. <br /> <br /> Only an end-to-end test can catch these integration issues, and we already had <a href="http://googletesting.blogspot.com/2013/11/webrtc-audio-quality-testing.html">stable, low-maintenance audio quality tests</a> with the ability to record Chrome’s output sound for analysis. I encourage you to read that article, but the bottom line is that those tests can run a WebRTC call in two tabs and record the audio output to a file. Those tests run the <a href="https://en.wikipedia.org/wiki/PESQ">PESQ</a> algorithm on input and output to see how similar they are. <br /> <br /> That’s a good framework to have, but I needed to make two changes:<br /> <br /> <ul> <li>Add file support to Chrome’s fake audio input device, so we can play a known file. The original audio test avoided this by using WebAudio, but AGC doesn’t run in the WebAudio path, just the microphone capture path, so that won’t work.</li> <li>Instead of running PESQ, run an analysis that compares the gain between input and output.</li> </ul> <h2> Adding Fake File Support</h2> This is always a big part of the work in media testing: controlling the input and output. It’s unworkable to tape microphones to loudspeakers or point cameras to screens to capture the media, so the easiest solution is usually to add a debug flag. It is exactly what I did <a href="https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/fake_audio_input_stream.cc&sq=package:chromium&type=cs&rcl=1443324274&l=118">here</a>. It was a lot of work, but I won’t go into much detail since Chrome’s audio pipeline is complex. The core is <a href="https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/simple_sources.cc&sq=package:chromium&type=cs">this</a>: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">int</span> FileSource::OnMoreData(AudioBus* audio_bus, uint32 total_bytes_delay) { <span style="color: #4f8f00;">// Load the file if we haven't already. This load needs to happen on the</span> <span style="color: #4f8f00;">// audio thread, otherwise we'll run on the UI thread on Mac for instance.</span> <span style="color: #4f8f00;">// This will massively delay the first OnMoreData, but we'll catch up.</span> <span style="color: #0433ff;">if</span> (!wav_audio_handler_) LoadWavFile(path_to_wav_file_); <span style="color: #0433ff;">if</span> (load_failed_) <span style="color: #0433ff;">return</span> 0; DCHECK(wav_audio_handler_.get()); <span style="color: #4f8f00;">// Stop playing if we've played out the whole file.</span> <span style="color: #0433ff;">if</span> (wav_audio_handler_->AtEnd(wav_file_read_pos_)) <span style="color: #0433ff;">return</span> 0; <span style="color: #4f8f00;">// This pulls data from ProvideInput.</span> file_audio_converter_->Convert(audio_bus); <span style="color: #0433ff;">return</span> audio_bus->frames(); } </pre> <br /> This code runs every 10 ms and reads a small chunk from the file, converts it to Chrome’s preferred audio format and sends it on through the audio pipeline. After implementing this, I could simply run: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">chrome --use-fake-device-for-media-stream \ --use-file-for-fake-audio-capture=/tmp/file.wav </pre> <br /> and whenever I hit a webpage that used WebRTC, the above file would play instead of my microphone input. Sweet!<br /> <h2> The Analysis Stage</h2> Next I had to get the analysis stage figured out. It turned out there was something called an <a href="https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/audio_power_monitor.h&q=audiopowermo&sq=package:chromium&l=36">AudioPowerMonitor</a> in the Chrome code, which you feed audio data into and get the average audio power for the data you fed in. This is a measure of how “loud” the audio is. Since the whole point of the AGC is getting to the right audio power level, we’re looking to compute<br /> <br /> <i>A<sub>diff</sub> = A<sub>out</sub> - A<sub>in</sub></i> <br /> <br /> Or, really, how much louder or weaker is the output compared to the input audio? Then we can construct different scenarios: <i>A<sub>diff</sub></i> should be 0 if the AGC is turned off and it should be > 0 dB if the AGC is on and we feed in a low power audio file. Computing the average energy of an audio file was straightforward to <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/webrtc_browsertest_audio.cc&sq=package:chromium&type=cs&l=56">implement</a>: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> <span style="color: #4f8f00;">// ...</span> size_t bytes_written; wav_audio_handler->CopyTo(audio_bus.get(), 0, &bytes_written); CHECK_EQ(bytes_written, wav_audio_handler->data().size()) << <span style="color: #ff2600;">"Expected to write entire file into bus."</span>; <span style="color: #4f8f00;">// Set the filter coefficient to the whole file's duration; this will make</span> <span style="color: #4f8f00;">// the power monitor take the entire file into account.</span> media::AudioPowerMonitor power_monitor(wav_audio_handler->sample_rate(), file_duration); power_monitor.Scan(*audio_bus, audio_bus->frames()); <span style="color: #4f8f00;">// ...</span> <span style="color: #0433ff;">return</span> power_monitor.ReadCurrentPowerAndClip().first;</pre> <br /> I wrote a new test, and hooked up the above logic instead of PESQ. I could compute <i>A<sub>in</sub></i> by running the above algorithm on the reference file (which I fed in using the flag I implemented above) and <i>A<sub>out</sub></i> on the recording of the output audio. At this point I pretty much thought I was done. I ran a WebRTC call with the AGC turned off, expecting to get zero… and got a huge number. Turns out I wasn’t done.<br /> <h2> What Went Wrong?</h2> I needed more debugging information to figure out what went wrong. Since the AGC was off, I would expect the power curves for output and input to be identical. All I had was the average audio power over the entire file, so I started plotting the audio power for each 10 millisecond segment instead to understand where the curves diverged. I could then plot the detected audio power over the time of the test. I started by plotting <i>A<sub>diff </sub></i>: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgKS4JJuCfqUNwZejtRf4UtNJrOuahKLbsQGQAB7Y0iTKqWTQgwqqcFPz4vdWAz-OqhpJtsq9j6VjgIInCtLcSZUJxEW05F92yZYFR0eG52NvucHSG5wmhfK1oyzkjjYUi3gsbO/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="161" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgKS4JJuCfqUNwZejtRf4UtNJrOuahKLbsQGQAB7Y0iTKqWTQgwqqcFPz4vdWAz-OqhpJtsq9j6VjgIInCtLcSZUJxEW05F92yZYFR0eG52NvucHSG5wmhfK1oyzkjjYUi3gsbO/s640/image01.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 2. Plot of A<sub>diff</sub>.</i></div> <br /> The difference is quite small in the beginning, but grows in amplitude over time. Interesting. I then plotted <i>A<sub>out</sub></i> and <i>A<sub>in</sub></i> next to each other: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgJKHtzNVExUPZn_ZtoD4mLrtiHMt169KSwdHocBg7q7ngPHBHPvtso1ToTeQPt_xRhzfBLqh6e8FL4nU4WadCOoJWuuBjA4wGqUrwtHlpdKOipV7fOsL6rCvwen9h-IkvlfyZs/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="249" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgJKHtzNVExUPZn_ZtoD4mLrtiHMt169KSwdHocBg7q7ngPHBHPvtso1ToTeQPt_xRhzfBLqh6e8FL4nU4WadCOoJWuuBjA4wGqUrwtHlpdKOipV7fOsL6rCvwen9h-IkvlfyZs/s640/image00.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 3. Plot of A<sub>out </sub>and A<sub>in</sub>.</i></div> <br /> A-ha! The curves drift apart over time; the above shows about 10 seconds of time, and the drift is maybe 80 ms at the end. The more they drift apart, the bigger the diff becomes. Exasperated, I asked our audio engineers about the above. Had my fancy test found its first bug? No, as it turns out - it was by design.<br /> <h2> Clock Drift and Packet Loss</h2> Let me explain. As a part of WebRTC audio processing, we run a complex module called <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_coding/neteq/&q=neteq&sq=package:chromium">NetEq</a> on the received audio stream. When sending audio over the Internet, there will inevitably be <i>packet loss</i> and <i>clock drift</i>. Packet losses always happen on the Internet, depending on the network path between sender and receiver. Clock drift happens because the sample clocks on the sending and receiving sound cards are not perfectly synced. <br /> <br /> In this particular case, the problem was not packet loss since we have ideal network conditions (one machine, packets go over the machine’s loopback interface = zero packet loss). But how can we have clock drift? Well, recall the fake device I wrote earlier that reads a file? It never touches the sound card like when the sound comes from the mic, so it runs on the system clock. That clock will drift against the machine’s sound card clock, even when we are on the same machine. <br /> <br /> NetEq uses clever algorithms to conceal clock drift and packet loss. Most commonly it applies time compression or stretching on the audio it plays out, which means it makes the audio a little shorter or longer when needed to compensate for the drift. We humans mostly don’t even notice that, whereas a drift left uncompensated would result in a depleted or flooded receiver buffer – very noticeable. Anyway, I digress. This drift of the recording vs. the reference file was natural and I would just have to deal with it.<br /> <h2> Silence Splitting to the Rescue!</h2> I could probably have solved this with math and postprocessing of the results (<a href="https://en.wikipedia.org/wiki/Least_squares">least squares</a> maybe?), but I had another idea. The reference file happened to be comprised of five segments with small pauses between them. What if I made these pauses longer, split the files on the pauses and trimmed away all the silence? This would effectively align the start of each segment with its corresponding segment in the reference file. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiuLcyOlJyhNKF1GBo4Uz7reXCf7B0bSr5t9ucWIJSWG-PLss58CZz_0rkmNW_kx933LdVJP16NR6B-Fer-l7Or2PsyvSOayEe4J8dOv-v9PYW7_ds_qtPNWHNAuVIq2evHTkEb/s1600/image05.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="112" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiuLcyOlJyhNKF1GBo4Uz7reXCf7B0bSr5t9ucWIJSWG-PLss58CZz_0rkmNW_kx933LdVJP16NR6B-Fer-l7Or2PsyvSOayEe4J8dOv-v9PYW7_ds_qtPNWHNAuVIq2evHTkEb/s640/image05.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 4. Before silence splitting.</i></div> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEje0eZEGjTJe3rS5HvPK71Ks1AgbsLbWfPojdKUTrSjH_zlEseOnMhuakuV-t87pPN-6gjbn4USRx_JKne0NVctG2X49OfVHAtjYFQht1G8oR3xAbRnIrE2m08uXQREt_2BJVrg/s1600/image04.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="176" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEje0eZEGjTJe3rS5HvPK71Ks1AgbsLbWfPojdKUTrSjH_zlEseOnMhuakuV-t87pPN-6gjbn4USRx_JKne0NVctG2X49OfVHAtjYFQht1G8oR3xAbRnIrE2m08uXQREt_2BJVrg/s640/image04.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 5. After silence splitting.</i></div> <br /> We would still have NetEQ drift, but as you can see its effects will not stack up towards the end, so if the segments are short enough we should be able to mitigate this problem.<br /> <h2> Result</h2> Here is the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc">final test implementation</a>: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> base::FilePath reference_file = test::GetReferenceFilesDir().Append(reference_filename); base::FilePath recording = CreateTemporaryWaveFile(); ASSERT_NO_FATAL_FAILURE(SetupAndRecordAudioCall( reference_file, recording, constraints, base::TimeDelta::FromSeconds(30))); base::ScopedTempDir split_ref_files; ASSERT_TRUE(split_ref_files.CreateUniqueTempDir()); ASSERT_NO_FATAL_FAILURE( SplitFileOnSilenceIntoDir(reference_file, split_ref_files.path())); std::<span style="color: #0433ff;">vector</span><base::FilePath> ref_segments = ListWavFilesInDir(split_ref_files.path()); base::ScopedTempDir split_actual_files; ASSERT_TRUE(split_actual_files.CreateUniqueTempDir()); ASSERT_NO_FATAL_FAILURE( SplitFileOnSilenceIntoDir(recording, split_actual_files.path())); <span style="color: #4f8f00;">// Keep the recording and split files if the analysis fails.</span> base::FilePath actual_files_dir = split_actual_files.Take(); std::<span style="color: #0433ff;">vector</span><base::FilePath> actual_segments = ListWavFilesInDir(actual_files_dir); AnalyzeSegmentsAndPrintResult( ref_segments, actual_segments, reference_file, perf_modifier); DeleteFileUnlessTestFailed(recording, <span style="color: #0433ff;">false</span>); DeleteFileUnlessTestFailed(actual_files_dir, <span style="color: #0433ff;">true</span>);</pre> <br /> Where AnalyzeSegmentsAndPrintResult looks like this: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">void</span> AnalyzeSegmentsAndPrintResult( <span style="color: #0433ff;">const</span> std::<span style="color: #0433ff;">vector</span><base::FilePath>& ref_segments, <span style="color: #0433ff;">const</span> std::<span style="color: #0433ff;">vector</span><base::FilePath>& actual_segments, <span style="color: #0433ff;">const</span> base::FilePath& reference_file, <span style="color: #0433ff;">const</span> std::<span style="color: #0433ff;">string</span>& perf_modifier) { ASSERT_GT(ref_segments.size(), 0u) << <span style="color: #ff2600;">"Failed to split reference file on silence; sox is likely broken."</span>; ASSERT_EQ(ref_segments.size(), actual_segments.size()) << <span style="color: #ff2600;">"The recording did not result in the same number of audio segments "</span> << <span style="color: #ff2600;">"after on splitting on silence; WebRTC must have deformed the audio "</span> << <span style="color: #ff2600;">"too much."</span>; <span style="color: #0433ff;">for</span> (size_t i = 0; i < ref_segments.size(); i++) { <span style="color: #0433ff;">float</span> difference_in_decibel = AnalyzeOneSegment(ref_segments[i], actual_segments[i], i); std::<span style="color: #0433ff;">string</span> trace_name = MakeTraceName(reference_file, i); perf_test::PrintResult(<span style="color: #ff2600;">"agc_energy_diff"</span>, perf_modifier, trace_name, difference_in_decibel, <span style="color: #ff2600;">"dB"</span>, <span style="color: #0433ff;">false</span>); } } </pre> <br /> The results look like <a href="https://chromeperf.appspot.com/report?sid=e7143debc512e03aa4b7a0358103e6635a8c71298a79192300034b2318f7017e">this</a>: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhLd-fDbARnc8ZS2M322u4jfs_ju6UiAxr6ke0MzOcF5B30HikVtFx200EEwSidaBLbq0UY6xR3Y36FJUqh3eTIBaQOS4PM62B6pDFMk6xVHercrBgHta6Y_JSDlpkwmswJ21_M/s1600/image02.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="321" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhLd-fDbARnc8ZS2M322u4jfs_ju6UiAxr6ke0MzOcF5B30HikVtFx200EEwSidaBLbq0UY6xR3Y36FJUqh3eTIBaQOS4PM62B6pDFMk6xVHercrBgHta6Y_JSDlpkwmswJ21_M/s640/image02.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 6. Average Adiff values for each segment on the y axis, Chromium revisions on the x axis.</i></div> <br /> We can clearly see the AGC applies about 6 dB of gain to the (relatively low-energy) audio file we feed in. The maximum amount of gain the digital AGC can apply is 12 dB, and 7 dB is the default, so in this case the AGC is pretty happy with the level of the input audio. If we run with the AGC turned off, we get the expected 0 dB of gain. The diff varies a bit per segment, since the segments are different in audio power. <br /> <br /> Using this test, we can detect if the AGC accidentally gets turned off or malfunctions on windows, mac or linux. If that happens, the with_agc graph will drop from ~6 db to 0, and we’ll know something is up. Same thing if the amount of digital gain changes. <br /> <br /> A more advanced version of this test would also look at the mic level the AGC sets. This mic level is currently ignored in the test, but it could take it into account by artificially amplifying the reference file when played through the fake device. We could also try throwing curveballs at the AGC, like abruptly raising the volume mid-test (as if the user leaned closer to the mic), and look at the gain for the segments to ensure it adapted correctly. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>By: Patrik Höglund </i><br /> <br /> <h2> What is Automatic Gain Control? </h2> It’s time to talk about advanced media quality tests again! As experienced Google testing blog readers know, when I write an article it’s usually about WebRTC, and the unusual testing solutions we build to test it. This article is no exception. Today we’re going to talk about Automatic Gain Control, or AGC. This is a feature that’s on by default for WebRTC applications, such as <a href="http://apprtc.appspot.com/">http://apprtc.appspot.com</a>. It uses various means to adjust the microphone signal so your voice makes it loud and clear to the other side of the peer connection. For instance, it can attempt to adjust your microphone gain or try to amplify the signal digitally.<br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhm7lOY1ZMr0tyR49jxejFz4IsseLaVb67h8akatZjCEs8pIRZFjhH4-980o0YjZ_ZzRuvyPzIfch-V3CgPIBoH8pqrkhAeW6GNRqv6K9uIZkU1EOM6rvbl9BvT5Zo1gMgArCEs/s1600/image03.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="336" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhm7lOY1ZMr0tyR49jxejFz4IsseLaVb67h8akatZjCEs8pIRZFjhH4-980o0YjZ_ZzRuvyPzIfch-V3CgPIBoH8pqrkhAeW6GNRqv6K9uIZkU1EOM6rvbl9BvT5Zo1gMgArCEs/s640/image03.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 1. How Auto Gain Control works [<a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_processing/agc/agc_manager_direct.h&sq=package:chromium">code here</a>]. </i></div> <div style="text-align: center;"> <br /></div> This is an example of automatic control engineering (another example would be the classic <a href="https://en.wikipedia.org/wiki/PID_controller">PID controller</a>) and happens in real time. Therefore, if you move closer to the mic while speaking, the AGC will notice the output stream is too loud, and reduce mic volume and/or digital gain. When you move further away, it tries to adapt up again. The fancy voice activity detector is there so we only amplify speech, and not, say, the microwave oven your spouse just started in the other room.<br /> <h2> Testing the AGC</h2> Now, how do we make sure the AGC works? The first thing is obviously to <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_processing/agc/agc_unittest.cc&sq=package:chromium">write unit tests</a> and <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_processing/test/">integration tests</a>. You <a href="http://googletesting.blogspot.com/2015/04/just-say-no-to-more-end-to-end-tests.html">didn’t think about building that end-to-end test first, did you</a>? Once we have the lower-level tests in place, we can start looking at a bigger test. While developing the WebRTC implementation in Chrome, we had several bugs where the AGC code was working by itself, but was misconfigured in Chrome. In one case, it was simply turned off for all users. In another, it was only turned off in Hangouts. <br /> <br /> Only an end-to-end test can catch these integration issues, and we already had <a href="http://googletesting.blogspot.com/2013/11/webrtc-audio-quality-testing.html">stable, low-maintenance audio quality tests</a> with the ability to record Chrome’s output sound for analysis. I encourage you to read that article, but the bottom line is that those tests can run a WebRTC call in two tabs and record the audio output to a file. Those tests run the <a href="https://en.wikipedia.org/wiki/PESQ">PESQ</a> algorithm on input and output to see how similar they are. <br /> <br /> That’s a good framework to have, but I needed to make two changes:<br /> <br /> <ul> <li>Add file support to Chrome’s fake audio input device, so we can play a known file. The original audio test avoided this by using WebAudio, but AGC doesn’t run in the WebAudio path, just the microphone capture path, so that won’t work.</li> <li>Instead of running PESQ, run an analysis that compares the gain between input and output.</li> </ul> <h2> Adding Fake File Support</h2> This is always a big part of the work in media testing: controlling the input and output. It’s unworkable to tape microphones to loudspeakers or point cameras to screens to capture the media, so the easiest solution is usually to add a debug flag. It is exactly what I did <a href="https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/fake_audio_input_stream.cc&sq=package:chromium&type=cs&rcl=1443324274&l=118">here</a>. It was a lot of work, but I won’t go into much detail since Chrome’s audio pipeline is complex. The core is <a href="https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/simple_sources.cc&sq=package:chromium&type=cs">this</a>: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">int</span> FileSource::OnMoreData(AudioBus* audio_bus, uint32 total_bytes_delay) { <span style="color: #4f8f00;">// Load the file if we haven't already. This load needs to happen on the</span> <span style="color: #4f8f00;">// audio thread, otherwise we'll run on the UI thread on Mac for instance.</span> <span style="color: #4f8f00;">// This will massively delay the first OnMoreData, but we'll catch up.</span> <span style="color: #0433ff;">if</span> (!wav_audio_handler_) LoadWavFile(path_to_wav_file_); <span style="color: #0433ff;">if</span> (load_failed_) <span style="color: #0433ff;">return</span> 0; DCHECK(wav_audio_handler_.get()); <span style="color: #4f8f00;">// Stop playing if we've played out the whole file.</span> <span style="color: #0433ff;">if</span> (wav_audio_handler_->AtEnd(wav_file_read_pos_)) <span style="color: #0433ff;">return</span> 0; <span style="color: #4f8f00;">// This pulls data from ProvideInput.</span> file_audio_converter_->Convert(audio_bus); <span style="color: #0433ff;">return</span> audio_bus->frames(); } </pre> <br /> This code runs every 10 ms and reads a small chunk from the file, converts it to Chrome’s preferred audio format and sends it on through the audio pipeline. After implementing this, I could simply run: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">chrome --use-fake-device-for-media-stream \ --use-file-for-fake-audio-capture=/tmp/file.wav </pre> <br /> and whenever I hit a webpage that used WebRTC, the above file would play instead of my microphone input. Sweet!<br /> <h2> The Analysis Stage</h2> Next I had to get the analysis stage figured out. It turned out there was something called an <a href="https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/audio_power_monitor.h&q=audiopowermo&sq=package:chromium&l=36">AudioPowerMonitor</a> in the Chrome code, which you feed audio data into and get the average audio power for the data you fed in. This is a measure of how “loud” the audio is. Since the whole point of the AGC is getting to the right audio power level, we’re looking to compute<br /> <br /> <i>A<sub>diff</sub> = A<sub>out</sub> - A<sub>in</sub></i> <br /> <br /> Or, really, how much louder or weaker is the output compared to the input audio? Then we can construct different scenarios: <i>A<sub>diff</sub></i> should be 0 if the AGC is turned off and it should be > 0 dB if the AGC is on and we feed in a low power audio file. Computing the average energy of an audio file was straightforward to <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/webrtc_browsertest_audio.cc&sq=package:chromium&type=cs&l=56">implement</a>: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> <span style="color: #4f8f00;">// ...</span> size_t bytes_written; wav_audio_handler->CopyTo(audio_bus.get(), 0, &bytes_written); CHECK_EQ(bytes_written, wav_audio_handler->data().size()) << <span style="color: #ff2600;">"Expected to write entire file into bus."</span>; <span style="color: #4f8f00;">// Set the filter coefficient to the whole file's duration; this will make</span> <span style="color: #4f8f00;">// the power monitor take the entire file into account.</span> media::AudioPowerMonitor power_monitor(wav_audio_handler->sample_rate(), file_duration); power_monitor.Scan(*audio_bus, audio_bus->frames()); <span style="color: #4f8f00;">// ...</span> <span style="color: #0433ff;">return</span> power_monitor.ReadCurrentPowerAndClip().first;</pre> <br /> I wrote a new test, and hooked up the above logic instead of PESQ. I could compute <i>A<sub>in</sub></i> by running the above algorithm on the reference file (which I fed in using the flag I implemented above) and <i>A<sub>out</sub></i> on the recording of the output audio. At this point I pretty much thought I was done. I ran a WebRTC call with the AGC turned off, expecting to get zero… and got a huge number. Turns out I wasn’t done.<br /> <h2> What Went Wrong?</h2> I needed more debugging information to figure out what went wrong. Since the AGC was off, I would expect the power curves for output and input to be identical. All I had was the average audio power over the entire file, so I started plotting the audio power for each 10 millisecond segment instead to understand where the curves diverged. I could then plot the detected audio power over the time of the test. I started by plotting <i>A<sub>diff </sub></i>: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgKS4JJuCfqUNwZejtRf4UtNJrOuahKLbsQGQAB7Y0iTKqWTQgwqqcFPz4vdWAz-OqhpJtsq9j6VjgIInCtLcSZUJxEW05F92yZYFR0eG52NvucHSG5wmhfK1oyzkjjYUi3gsbO/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="161" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgKS4JJuCfqUNwZejtRf4UtNJrOuahKLbsQGQAB7Y0iTKqWTQgwqqcFPz4vdWAz-OqhpJtsq9j6VjgIInCtLcSZUJxEW05F92yZYFR0eG52NvucHSG5wmhfK1oyzkjjYUi3gsbO/s640/image01.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 2. Plot of A<sub>diff</sub>.</i></div> <br /> The difference is quite small in the beginning, but grows in amplitude over time. Interesting. I then plotted <i>A<sub>out</sub></i> and <i>A<sub>in</sub></i> next to each other: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgJKHtzNVExUPZn_ZtoD4mLrtiHMt169KSwdHocBg7q7ngPHBHPvtso1ToTeQPt_xRhzfBLqh6e8FL4nU4WadCOoJWuuBjA4wGqUrwtHlpdKOipV7fOsL6rCvwen9h-IkvlfyZs/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="249" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgJKHtzNVExUPZn_ZtoD4mLrtiHMt169KSwdHocBg7q7ngPHBHPvtso1ToTeQPt_xRhzfBLqh6e8FL4nU4WadCOoJWuuBjA4wGqUrwtHlpdKOipV7fOsL6rCvwen9h-IkvlfyZs/s640/image00.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 3. Plot of A<sub>out </sub>and A<sub>in</sub>.</i></div> <br /> A-ha! The curves drift apart over time; the above shows about 10 seconds of time, and the drift is maybe 80 ms at the end. The more they drift apart, the bigger the diff becomes. Exasperated, I asked our audio engineers about the above. Had my fancy test found its first bug? No, as it turns out - it was by design.<br /> <h2> Clock Drift and Packet Loss</h2> Let me explain. As a part of WebRTC audio processing, we run a complex module called <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc/modules/audio_coding/neteq/&q=neteq&sq=package:chromium">NetEq</a> on the received audio stream. When sending audio over the Internet, there will inevitably be <i>packet loss</i> and <i>clock drift</i>. Packet losses always happen on the Internet, depending on the network path between sender and receiver. Clock drift happens because the sample clocks on the sending and receiving sound cards are not perfectly synced. <br /> <br /> In this particular case, the problem was not packet loss since we have ideal network conditions (one machine, packets go over the machine’s loopback interface = zero packet loss). But how can we have clock drift? Well, recall the fake device I wrote earlier that reads a file? It never touches the sound card like when the sound comes from the mic, so it runs on the system clock. That clock will drift against the machine’s sound card clock, even when we are on the same machine. <br /> <br /> NetEq uses clever algorithms to conceal clock drift and packet loss. Most commonly it applies time compression or stretching on the audio it plays out, which means it makes the audio a little shorter or longer when needed to compensate for the drift. We humans mostly don’t even notice that, whereas a drift left uncompensated would result in a depleted or flooded receiver buffer – very noticeable. Anyway, I digress. This drift of the recording vs. the reference file was natural and I would just have to deal with it.<br /> <h2> Silence Splitting to the Rescue!</h2> I could probably have solved this with math and postprocessing of the results (<a href="https://en.wikipedia.org/wiki/Least_squares">least squares</a> maybe?), but I had another idea. The reference file happened to be comprised of five segments with small pauses between them. What if I made these pauses longer, split the files on the pauses and trimmed away all the silence? This would effectively align the start of each segment with its corresponding segment in the reference file. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiuLcyOlJyhNKF1GBo4Uz7reXCf7B0bSr5t9ucWIJSWG-PLss58CZz_0rkmNW_kx933LdVJP16NR6B-Fer-l7Or2PsyvSOayEe4J8dOv-v9PYW7_ds_qtPNWHNAuVIq2evHTkEb/s1600/image05.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="112" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiuLcyOlJyhNKF1GBo4Uz7reXCf7B0bSr5t9ucWIJSWG-PLss58CZz_0rkmNW_kx933LdVJP16NR6B-Fer-l7Or2PsyvSOayEe4J8dOv-v9PYW7_ds_qtPNWHNAuVIq2evHTkEb/s640/image05.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 4. Before silence splitting.</i></div> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEje0eZEGjTJe3rS5HvPK71Ks1AgbsLbWfPojdKUTrSjH_zlEseOnMhuakuV-t87pPN-6gjbn4USRx_JKne0NVctG2X49OfVHAtjYFQht1G8oR3xAbRnIrE2m08uXQREt_2BJVrg/s1600/image04.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="176" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEje0eZEGjTJe3rS5HvPK71Ks1AgbsLbWfPojdKUTrSjH_zlEseOnMhuakuV-t87pPN-6gjbn4USRx_JKne0NVctG2X49OfVHAtjYFQht1G8oR3xAbRnIrE2m08uXQREt_2BJVrg/s640/image04.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 5. After silence splitting.</i></div> <br /> We would still have NetEQ drift, but as you can see its effects will not stack up towards the end, so if the segments are short enough we should be able to mitigate this problem.<br /> <h2> Result</h2> Here is the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc">final test implementation</a>: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> base::FilePath reference_file = test::GetReferenceFilesDir().Append(reference_filename); base::FilePath recording = CreateTemporaryWaveFile(); ASSERT_NO_FATAL_FAILURE(SetupAndRecordAudioCall( reference_file, recording, constraints, base::TimeDelta::FromSeconds(30))); base::ScopedTempDir split_ref_files; ASSERT_TRUE(split_ref_files.CreateUniqueTempDir()); ASSERT_NO_FATAL_FAILURE( SplitFileOnSilenceIntoDir(reference_file, split_ref_files.path())); std::<span style="color: #0433ff;">vector</span><base::FilePath> ref_segments = ListWavFilesInDir(split_ref_files.path()); base::ScopedTempDir split_actual_files; ASSERT_TRUE(split_actual_files.CreateUniqueTempDir()); ASSERT_NO_FATAL_FAILURE( SplitFileOnSilenceIntoDir(recording, split_actual_files.path())); <span style="color: #4f8f00;">// Keep the recording and split files if the analysis fails.</span> base::FilePath actual_files_dir = split_actual_files.Take(); std::<span style="color: #0433ff;">vector</span><base::FilePath> actual_segments = ListWavFilesInDir(actual_files_dir); AnalyzeSegmentsAndPrintResult( ref_segments, actual_segments, reference_file, perf_modifier); DeleteFileUnlessTestFailed(recording, <span style="color: #0433ff;">false</span>); DeleteFileUnlessTestFailed(actual_files_dir, <span style="color: #0433ff;">true</span>);</pre> <br /> Where AnalyzeSegmentsAndPrintResult looks like this: <br /> <br /> <pre style="background: #d6d6d6; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">void</span> AnalyzeSegmentsAndPrintResult( <span style="color: #0433ff;">const</span> std::<span style="color: #0433ff;">vector</span><base::FilePath>& ref_segments, <span style="color: #0433ff;">const</span> std::<span style="color: #0433ff;">vector</span><base::FilePath>& actual_segments, <span style="color: #0433ff;">const</span> base::FilePath& reference_file, <span style="color: #0433ff;">const</span> std::<span style="color: #0433ff;">string</span>& perf_modifier) { ASSERT_GT(ref_segments.size(), 0u) << <span style="color: #ff2600;">"Failed to split reference file on silence; sox is likely broken."</span>; ASSERT_EQ(ref_segments.size(), actual_segments.size()) << <span style="color: #ff2600;">"The recording did not result in the same number of audio segments "</span> << <span style="color: #ff2600;">"after on splitting on silence; WebRTC must have deformed the audio "</span> << <span style="color: #ff2600;">"too much."</span>; <span style="color: #0433ff;">for</span> (size_t i = 0; i < ref_segments.size(); i++) { <span style="color: #0433ff;">float</span> difference_in_decibel = AnalyzeOneSegment(ref_segments[i], actual_segments[i], i); std::<span style="color: #0433ff;">string</span> trace_name = MakeTraceName(reference_file, i); perf_test::PrintResult(<span style="color: #ff2600;">"agc_energy_diff"</span>, perf_modifier, trace_name, difference_in_decibel, <span style="color: #ff2600;">"dB"</span>, <span style="color: #0433ff;">false</span>); } } </pre> <br /> The results look like <a href="https://chromeperf.appspot.com/report?sid=e7143debc512e03aa4b7a0358103e6635a8c71298a79192300034b2318f7017e">this</a>: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhLd-fDbARnc8ZS2M322u4jfs_ju6UiAxr6ke0MzOcF5B30HikVtFx200EEwSidaBLbq0UY6xR3Y36FJUqh3eTIBaQOS4PM62B6pDFMk6xVHercrBgHta6Y_JSDlpkwmswJ21_M/s1600/image02.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="321" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhLd-fDbARnc8ZS2M322u4jfs_ju6UiAxr6ke0MzOcF5B30HikVtFx200EEwSidaBLbq0UY6xR3Y36FJUqh3eTIBaQOS4PM62B6pDFMk6xVHercrBgHta6Y_JSDlpkwmswJ21_M/s640/image02.png" width="640" /></a></div> <div style="text-align: center;"> <i>Figure 6. Average Adiff values for each segment on the y axis, Chromium revisions on the x axis.</i></div> <br /> We can clearly see the AGC applies about 6 dB of gain to the (relatively low-energy) audio file we feed in. The maximum amount of gain the digital AGC can apply is 12 dB, and 7 dB is the default, so in this case the AGC is pretty happy with the level of the input audio. If we run with the AGC turned off, we get the expected 0 dB of gain. The diff varies a bit per segment, since the segments are different in audio power. <br /> <br /> Using this test, we can detect if the AGC accidentally gets turned off or malfunctions on windows, mac or linux. If that happens, the with_agc graph will drop from ~6 db to 0, and we’ll know something is up. Same thing if the amount of digital gain changes. <br /> <br /> A more advanced version of this test would also look at the mic level the AGC sets. This mic level is currently ignored in the test, but it could take it into account by artificially amplifying the reference file when played through the fake device. We could also try throwing curveballs at the AGC, like abruptly raising the volume mid-test (as if the user leaned closer to the mic), and look at the gain for the segments to ensure it adapted correctly. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Audio Testing - Automatic Gain Control&url=https://testing.googleblog.com/2015/10/audio-testing-automatic-gain-control.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/10/audio-testing-automatic-gain-control.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/10/audio-testing-automatic-gain-control.html#comments' style='font-weight: 500; text-decoration: underline;'>4 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/10/audio-testing-automatic-gain-control.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund' rel='tag'> Patrik Höglund </a> , <a class='label' href='https://testing.googleblog.com/search/label/WebRTC' rel='tag'> WebRTC </a> </span> </div> </div> </div> <div class='post' data-id='1654381234644274314' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/08/the-deadline-to-apply-for-gtac-2015-is.html' itemprop='url' title='The Deadline to Apply for GTAC 2015 is Monday Aug 10'> The Deadline to Apply for GTAC 2015 is Monday Aug 10 </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Friday, August 07, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgO9ZFmsbywtpikZOdkiyCgyb3rnrgqBC5i3cXSO1_Y0Onj5_4_qM0_4XVJUMt9O9Qt3pLqahDlgGVej1fsZrdFLqEvPwl1fxLkBHQwjFog61__pQkp6LVeMC9r-ttYKZ_N4Rol/s1600/gtac-small.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgO9ZFmsbywtpikZOdkiyCgyb3rnrgqBC5i3cXSO1_Y0Onj5_4_qM0_4XVJUMt9O9Qt3pLqahDlgGVej1fsZrdFLqEvPwl1fxLkBHQwjFog61__pQkp6LVeMC9r-ttYKZ_N4Rol/s1600/gtac-small.png" /></a></div> <br /> The deadline to apply for GTAC 2015 is this Monday, August 10th, 2015. There is a great deal of interest to both attend and speak, and we’ve received many outstanding proposals. However, it’s not too late to submit your proposal for consideration. If you would like to speak or attend, be sure to <a href="https://docs.google.com/a/google.com/forms/d/1fEByEl11ixnZ8U-fd_iwJgs4tt5eGPIPisLLRWMiBK0/viewform">complete the form</a> by Monday. <br /> <br /> We will be making regular updates to the GTAC site (<a href="http://developers.google.com/gtac/2015">developers.google.com/gtac/2015/</a>) over the next several weeks, and you can find conference details there. <br /> <br /> For those that have already signed up to attend or speak, we will contact you directly by mid-September. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgO9ZFmsbywtpikZOdkiyCgyb3rnrgqBC5i3cXSO1_Y0Onj5_4_qM0_4XVJUMt9O9Qt3pLqahDlgGVej1fsZrdFLqEvPwl1fxLkBHQwjFog61__pQkp6LVeMC9r-ttYKZ_N4Rol/s1600/gtac-small.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgO9ZFmsbywtpikZOdkiyCgyb3rnrgqBC5i3cXSO1_Y0Onj5_4_qM0_4XVJUMt9O9Qt3pLqahDlgGVej1fsZrdFLqEvPwl1fxLkBHQwjFog61__pQkp6LVeMC9r-ttYKZ_N4Rol/s1600/gtac-small.png" /></a></div> <br /> The deadline to apply for GTAC 2015 is this Monday, August 10th, 2015. There is a great deal of interest to both attend and speak, and we’ve received many outstanding proposals. However, it’s not too late to submit your proposal for consideration. If you would like to speak or attend, be sure to <a href="https://docs.google.com/a/google.com/forms/d/1fEByEl11ixnZ8U-fd_iwJgs4tt5eGPIPisLLRWMiBK0/viewform">complete the form</a> by Monday. <br /> <br /> We will be making regular updates to the GTAC site (<a href="http://developers.google.com/gtac/2015">developers.google.com/gtac/2015/</a>) over the next several weeks, and you can find conference details there. <br /> <br /> For those that have already signed up to attend or speak, we will contact you directly by mid-September. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:The Deadline to Apply for GTAC 2015 is Monday Aug 10&url=https://testing.googleblog.com/2015/08/the-deadline-to-apply-for-gtac-2015-is.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/08/the-deadline-to-apply-for-gtac-2015-is.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/08/the-deadline-to-apply-for-gtac-2015-is.html#comments' style='font-weight: 500; text-decoration: underline;'>3 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/08/the-deadline-to-apply-for-gtac-2015-is.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </span> </div> </div> </div> <div class='post' data-id='8919440380173971880' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/06/gtac-2015-call-for-proposals-attendance.html' itemprop='url' title='GTAC 2015: Call for Proposals & Attendance'> GTAC 2015: Call for Proposals & Attendance </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, June 30, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> The GTAC (Google Test Automation Conference) 2015 application process is now open for presentation proposals and attendance. GTAC will be held at the <a href="https://www.google.com/about/careers/locations/cambridge/">Google Cambridge office</a> (near Boston, Massachusetts, USA) on November 10th - 11th, 2015. <br /> <br /> GTAC will be streamed live on YouTube again this year, so even if you can’t attend in person, you’ll be able to watch the conference remotely. We will post the live stream information as we get closer to the event, and recordings will be posted afterward. <br /> <br /> <b>Speakers</b><br /> Presentations are targeted at student, academic, and experienced engineers working on test automation. Full presentations are 30 minutes and lightning talks are 10 minutes. Speakers should be prepared for a question and answer session following their presentation. <br /> <br /> <b>Application</b><br /> For presentation proposals and/or attendance, <a href="https://docs.google.com/a/google.com/forms/d/1fEByEl11ixnZ8U-fd_iwJgs4tt5eGPIPisLLRWMiBK0/viewform">complete this form</a>. We will be selecting about 25 talks and 200 attendees for the event. The selection process is not first come first serve (no need to rush your application), and we select a diverse group of engineers from various locations, company sizes, and technical backgrounds (academic, industry expert, junior engineer, etc). <br /> <br /> <b>Deadline</b><br /> The due date for both presentation and attendance applications is August 10th, 2015. <br /> <br /> <b>Fees</b><br /> There are no registration fees, but speakers and attendees must arrange and pay for their own travel and accommodations. <br /> <br /> <b>More information</b><br /> You can find more details at <a href="http://developers.google.com/gtac">developers.google.com/gtac</a>. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> The GTAC (Google Test Automation Conference) 2015 application process is now open for presentation proposals and attendance. GTAC will be held at the <a href="https://www.google.com/about/careers/locations/cambridge/">Google Cambridge office</a> (near Boston, Massachusetts, USA) on November 10th - 11th, 2015. <br /> <br /> GTAC will be streamed live on YouTube again this year, so even if you can’t attend in person, you’ll be able to watch the conference remotely. We will post the live stream information as we get closer to the event, and recordings will be posted afterward. <br /> <br /> <b>Speakers</b><br /> Presentations are targeted at student, academic, and experienced engineers working on test automation. Full presentations are 30 minutes and lightning talks are 10 minutes. Speakers should be prepared for a question and answer session following their presentation. <br /> <br /> <b>Application</b><br /> For presentation proposals and/or attendance, <a href="https://docs.google.com/a/google.com/forms/d/1fEByEl11ixnZ8U-fd_iwJgs4tt5eGPIPisLLRWMiBK0/viewform">complete this form</a>. We will be selecting about 25 talks and 200 attendees for the event. The selection process is not first come first serve (no need to rush your application), and we select a diverse group of engineers from various locations, company sizes, and technical backgrounds (academic, industry expert, junior engineer, etc). <br /> <br /> <b>Deadline</b><br /> The due date for both presentation and attendance applications is August 10th, 2015. <br /> <br /> <b>Fees</b><br /> There are no registration fees, but speakers and attendees must arrange and pay for their own travel and accommodations. <br /> <br /> <b>More information</b><br /> You can find more details at <a href="http://developers.google.com/gtac">developers.google.com/gtac</a>. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2015: Call for Proposals & Attendance&url=https://testing.googleblog.com/2015/06/gtac-2015-call-for-proposals-attendance.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/06/gtac-2015-call-for-proposals-attendance.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/06/gtac-2015-call-for-proposals-attendance.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/06/gtac-2015-call-for-proposals-attendance.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </span> </div> </div> </div> <div class='post' data-id='5861027017157279175' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/05/gtac-2015-coming-to-cambridge-greater.html' itemprop='url' title='GTAC 2015 Coming to Cambridge (Greater Boston) in November '> GTAC 2015 Coming to Cambridge (Greater Boston) in November </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Thursday, May 28, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEinzUiq64wJ3usIWRfRvf9Fh34uU7UHUZMC5ZqAS7LPWjnxSJu566c1qe4owbu85X-ZGWImk_zT70EyHEVxuWIETHpi30De6V5sHImRH0Qo5T6rkpn6KDiW-eIQQq6h3KHbwJYK/s1600/gtac-colored-letters.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="106" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEinzUiq64wJ3usIWRfRvf9Fh34uU7UHUZMC5ZqAS7LPWjnxSJu566c1qe4owbu85X-ZGWImk_zT70EyHEVxuWIETHpi30De6V5sHImRH0Qo5T6rkpn6KDiW-eIQQq6h3KHbwJYK/s320/gtac-colored-letters.png" width="320" /></a></div> <br /> We are pleased to announce that the ninth GTAC (Google Test Automation Conference) will be held in <a href="https://www.google.com/about/careers/locations/cambridge/">Cambridge</a> (Greatah Boston, USA) on November 10th and 11th (<a href="http://en.wikipedia.org/wiki/Boston_accent">Toozdee and Wenzdee</a>), 2015. So, tell everyone to save the date for this wicked good event. <br /> <br /> GTAC is an annual conference hosted by Google, bringing together engineers from industry and academia to discuss advances in test automation and the test engineering computer science field. It’s a great opportunity to present, learn, and challenge modern testing technologies and strategies. <br /> <br /> You can browse presentation abstracts, slides, and videos from previous years on the <a href="https://developers.google.com/google-test-automation-conference/">GTAC</a> site. <br /> <br /> Stay tuned to this blog and the GTAC website for application information and opportunities to present at GTAC. Subscribing to this blog is the best way to get notified. We're looking forward to seeing you there! <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee </i><br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEinzUiq64wJ3usIWRfRvf9Fh34uU7UHUZMC5ZqAS7LPWjnxSJu566c1qe4owbu85X-ZGWImk_zT70EyHEVxuWIETHpi30De6V5sHImRH0Qo5T6rkpn6KDiW-eIQQq6h3KHbwJYK/s1600/gtac-colored-letters.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="106" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEinzUiq64wJ3usIWRfRvf9Fh34uU7UHUZMC5ZqAS7LPWjnxSJu566c1qe4owbu85X-ZGWImk_zT70EyHEVxuWIETHpi30De6V5sHImRH0Qo5T6rkpn6KDiW-eIQQq6h3KHbwJYK/s320/gtac-colored-letters.png" width="320" /></a></div> <br /> We are pleased to announce that the ninth GTAC (Google Test Automation Conference) will be held in <a href="https://www.google.com/about/careers/locations/cambridge/">Cambridge</a> (Greatah Boston, USA) on November 10th and 11th (<a href="http://en.wikipedia.org/wiki/Boston_accent">Toozdee and Wenzdee</a>), 2015. So, tell everyone to save the date for this wicked good event. <br /> <br /> GTAC is an annual conference hosted by Google, bringing together engineers from industry and academia to discuss advances in test automation and the test engineering computer science field. It’s a great opportunity to present, learn, and challenge modern testing technologies and strategies. <br /> <br /> You can browse presentation abstracts, slides, and videos from previous years on the <a href="https://developers.google.com/google-test-automation-conference/">GTAC</a> site. <br /> <br /> Stay tuned to this blog and the GTAC website for application information and opportunities to present at GTAC. Subscribing to this blog is the best way to get notified. We're looking forward to seeing you there! <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2015 Coming to Cambridge (Greater Boston) in November &url=https://testing.googleblog.com/2015/05/gtac-2015-coming-to-cambridge-greater.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/05/gtac-2015-coming-to-cambridge-greater.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/05/gtac-2015-coming-to-cambridge-greater.html#comments' style='font-weight: 500; text-decoration: underline;'>31 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/05/gtac-2015-coming-to-cambridge-greater.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </span> </div> </div> </div> <div class='post' data-id='6017429491515355169' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/05/multi-repository-development.html' itemprop='url' title='Multi-Repository Development'> Multi-Repository Development </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Friday, May 15, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>Author: Patrik Höglund </i><br /> <br /> As we all know, software development is a complicated activity where we develop features and applications to provide value to our users. Furthermore, any nontrivial modern software is composed out of other software. For instance, the Chrome web browser pulls <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/&sq=package:chromium">roughly a hundred libraries</a> into its third_party folder when you build the browser. The most significant of these libraries is Blink, the rendering engine, but there’s also ffmpeg for image processing, skia for low-level 2D graphics, and WebRTC for real-time communication (to name a few). <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjD2L7vtCiPSjIHJ1CHBI-Uc1CGVq6_pSplgp47TUXYAZsYC7eHyl8SN9UnWr4ts06UKX7of3OYXWbyS9doxiyP5kgpbl7NBfw8TaXe3CEG8Mp2qCXBpWmPrQPlah6Dw4xIE1kK/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjD2L7vtCiPSjIHJ1CHBI-Uc1CGVq6_pSplgp47TUXYAZsYC7eHyl8SN9UnWr4ts06UKX7of3OYXWbyS9doxiyP5kgpbl7NBfw8TaXe3CEG8Mp2qCXBpWmPrQPlah6Dw4xIE1kK/s1600/image01.png" /></a></div> <div style="text-align: center;"> <i>Figure 1. Holy dependencies, Batman!</i></div> <br /> There are many reasons to use software libraries. Why write your own phone number parser when you can use <a href="https://github.com/googlei18n/libphonenumber">libphonenumber</a>, which is battle-tested by real use in Android and Chrome and available under a permissive license? Using such software frees you up to focus on the core of your software so you can deliver a unique experience to your users. On the other hand, you need to keep your application up to date with changes in the library (you want that latest bug fix, right?), and you also run a risk of such a change breaking your application. This article will examine that integration problem and how you can reduce the risks associated with it.<br /> <h2> Updating Dependencies is Hard</h2> The simplest solution is to check in a copy of the library, build with it, and avoid touching it as much as possible. This solution, however, can be problematic because you miss out on bug fixes and new features in the library. What if you need a new feature or bug fix that just made it in? You have a few options:<br /> <ul class="my-list"> <li>Update the library to its latest release. If it’s been a long time since you did this, it can be quite risky and you may have to spend significant testing resources to ensure all the accumulated changes don’t break your application. You may have to catch up to interface changes in the library as well. </li> <li>Cherry-pick the feature/bug fix you want into your copy of the library. This is even riskier because your cherry-picked patches may depend on other changes in the library in subtle ways. Also, you still are not up to date with the latest version. </li> <li>Find some way to make do without the feature or bug fix.</li> </ul> None of the above options are very good. Using this ad-hoc updating model can work if there’s a low volume of changes in the library and our requirements on the library don’t change very often. Even if that is the case, what will you do if a critical zero-day exploit is discovered in your socket library? <br /> <br /> One way to mitigate the update risk is to integrate more often with your dependencies. As an extreme example, let’s look at Chrome. <br /> <br /> In Chrome development, there’s a massive amount of change going into its dependencies. The Blink rendering engine lives in a separate code repository from the browser. Blink sees hundreds of code changes per day, and Chrome must integrate with Blink often since it’s an important part of the browser. Another example is the WebRTC implementation, where a large part of Chrome’s implementation resides in the webrtc.org repository. This article will focus on the latter because it’s the team I happen to work on.<br /> <h2> How “Rolling” Works </h2> The open-sourced WebRTC codebase is used by Chrome but also by a number of other companies working on WebRTC. Chrome uses a toolchain called depot_tools to manage dependencies, and there’s a checked-in text file called <a href="https://code.google.com/p/chromium/codesearch#chromium/src/DEPS&q=DEPS&sq=package:chromium&l=1">DEPS</a> where dependencies are managed. It looks roughly like this:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">{ <span style="color: #4f8f00;"># ... </span> <span style="color: #ff2600;">'src/third_party/webrtc'</span>: <span style="color: #ff2600;">'https://chromium.googlesource.com/'</span> + <span style="color: #ff2600;">'external/webrtc/trunk/webrtc.git'</span> + <span style="color: #ff2600;">'@'</span> + <span style="color: #ff2600;">'5727038f572c517204e1642b8bc69b25381c4e9f'</span>, } </pre> <br /> The above means we should pull WebRTC from the specified git repository at the <span style="font-family: Courier New, Courier, monospace;">572703...</span> hash, similar to other dependency-provisioning frameworks. To build Chrome with a new version, we change the hash and check in a new version of the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/DEPS&q=DEPS&sq=package:chromium&l=1">DEPS</a> file. If the library’s API has changed, we must update Chrome to use the new API in the same patch. This process is known as <i>rolling</i> WebRTC to a new version. <br /> <br /> Now the problem is that we have changed the code going into Chrome. Maybe <a href="https://code.google.com/p/chromium/issues/detail?id=430001">getUserMedia has started crashing on Android</a>, or maybe the browser no longer boots on Windows. We don’t know until we have built and run all the tests. Therefore a roll patch is subject to the same presubmit checks as any Chrome patch (i.e. <a href="https://www.chromium.org/developers/testing/try-server-usage">many tests, on all platforms we ship on</a>). However, roll patches can be considerably more painful and risky than other patches. <br /> <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTH4ArXq9A6nAaV-Sc_PNNafRrP3BL2GA3-BoYb0smhQwSUnk2xKPh2Nx0L6IPIi-O9gsljux6p_85Sr0Mww1jtUplbxOGRTN_Glad3gtpnm8r_HcMQ0FEJy3cJwZL7Fn6HzvK/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTH4ArXq9A6nAaV-Sc_PNNafRrP3BL2GA3-BoYb0smhQwSUnk2xKPh2Nx0L6IPIi-O9gsljux6p_85Sr0Mww1jtUplbxOGRTN_Glad3gtpnm8r_HcMQ0FEJy3cJwZL7Fn6HzvK/s1600/image00.png" /></a></div> <div style="text-align: center;"> <i>Figure 2. Life of a Roll Patch.</i></div> <br /> On the WebRTC team we found ourselves in an uncomfortable position a couple years back. Developers would make changes to the webrtc.org code and there was a fair amount of churn in the interface, which meant we would have to update Chrome to adapt to those changes. Also we frequently broke tests and WebRTC functionality in Chrome because semantic changes had unexpected consequences in Chrome. Since rolls were so risky and painful to make, they started to happen less often, which made things even worse. There could be two weeks between rolls, which meant Chrome was hit by a large number of changes in one patch.<br /> <h2> Bots That Can See the Future: “FYI Bots” </h2> We found a way to mitigate this which we called FYI (for your information) bots. A bot is Chrome lingo for a continuous build machine which builds Chrome and runs tests. <br /> <br /> All the existing Chrome bots at that point would build Chrome as specified in the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/DEPS&q=DEPS&sq=package:chromium&l=1">DEPS</a> file, which meant they would build the WebRTC version we had rolled to up to that point. FYI bots replace that pinned version with WebRTC HEAD, but otherwise build and run Chrome-level tests as usual. Therefore: <br /> <br /> <ul class="my-list"> <li>If all the FYI bots were green, we knew a roll most likely would go smoothly. </li> <li>If the bots didn’t compile, we knew we would have to adapt Chrome to an interface change in the next roll patch. </li> <li>If the bots were red, we knew we either had a bug in WebRTC or that Chrome would have to be adapted to some semantic change in WebRTC.</li> </ul> The <a href="https://build.chromium.org/p/chromium.webrtc.fyi/waterfall">FYI “waterfall”</a> (a set of bots that builds and runs tests) is a straight copy of the <a href="https://build.chromium.org/p/chromium.webrtc/waterfall">main waterfall</a>, which is expensive in resources. We could have cheated and just set up FYI bots for one platform (say, Linux), but the most expensive regressions are platform-specific, so we reckoned the extra machines and maintenance were worth it.<br /> <h2> Making Gradual Interface Changes </h2> This solution helped but wasn’t quite satisfactory. We initially had the policy that it was fine to break the FYI bots since we could not update Chrome to use a new interface until the new interface had actually been rolled into Chrome. This, however, often caused the FYI bots to be compile-broken for days. We quickly started to suffer from <a href="http://www.martinfowler.com/articles/continuousIntegration.html#FixBrokenBuildsImmediately">red blindness</a> <a href="#ref1">[1]</a> and had no idea if we would break tests on the roll, especially if an interface change was made early in the roll cycle. <br /> <br /> The solution was to move to a more careful update policy for the WebRTC API. For the more technically inclined, “careful” here means “following the <a href="https://wiki.eclipse.org/Evolving_Java-based_APIs#API_Prime_Directive">API prime directive</a>” <a href="#ref2">[2]</a>. Consider this example:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">class</span> WebRtcAmplifier { ... <span style="color: #0433ff;">int</span> SetOutputVolume(<span style="color: #0433ff;">float</span> volume); }</pre> <br /> Normally we would just change the method’s signature when we needed to:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">class</span> WebRtcAmplifier { ... <span style="color: #0433ff;">int</span> SetOutputVolume(<span style="color: #0433ff;">float</span> volume, <span style="color: #0433ff;">bool</span> allow_eleven<a href="#foot1"><sup>1</sup></a>); }</pre> <br /> … but this would compile-break Chome until it could be updated. So we started doing it like this instead:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">class</span> WebRtcAmplifier { ... <span style="color: #0433ff;">int</span> SetOutputVolume(<span style="color: #0433ff;">float</span> volume); <span style="color: #0433ff;">int</span> SetOutputVolume2(<span style="color: #0433ff;">float</span> volume, <span style="color: #0433ff;">bool</span> allow_eleven); }</pre> <br /> Then we could:<br /> <ol class="my-list"> <li>Roll into Chrome </li> <li>Make Chrome use SetOutputVolume2 </li> <li>Update SetOutputVolume’s signature </li> <li>Roll again and make Chrome use SetOutputVolume </li> <li>Delete SetOutputVolume2</li> </ol> This approach requires several steps but we end up with the right interface and at no point do we break Chrome.<br /> <h2> Results</h2> When we implemented the above, we could fix problems as they came up rather than in big batches on each roll. We could institute the policy that the FYI bots should always be green, and that changes breaking them should be immediately rolled back. This made a huge difference. The team could work smoother and roll more often. This reduced our risk quite a bit, particularly when Chrome was about to cut a new version branch. Instead of doing panicked and risky rolls around a release, we could work out issues in good time and stay in control.<br /> <br /> Another benefit of FYI bots is more granular <a href="https://chromeperf.appspot.com/report?masters=ChromiumWebRTCFYI&bots=chromium-webrtc-trunk-tot-rel-linux&tests=browser_tests%2Fagc_energy_diff_with_agc&checked=all">performance tests</a>. Before the FYI bots, it would frequently happen that a bunch of metrics regressed. However, it’s not fun to find which of the 100 patches in the roll caused the regression! With the FYI bots, we can see precisely which WebRTC revision caused the problem.<br /> <h2> Future Work: Optimistic Auto-rolling</h2> The final step on this ladder (short of actually merging the repositories) is auto-rolling. The Blink team implemented this with their <a href="https://www.chromium.org/blink/blinkrollbot">ARB (AutoRollBot)</a>. The bot wakes up periodically and tries to do a roll patch. If it fails on the trybots, it waits and tries again later (perhaps the trybots failed because of a flake or other temporary error, or perhaps the error was real but has been fixed).<br /> <br /> To pull auto-rolling off, you are going to need very good tests. That goes for any roll patch (or any patch, really), but if you’re edging closer to a release and an unstoppable flood of code changes keep breaking you, you’re not in a good place.<br /> <br /> <hr /> <h2> References</h2> <a name="ref1"></a>[1] Martin Fowler (May 2006) “Continuous Integration”<br /> <a name="ref2"></a>[2] Dani Megert, Remy Chi Jian Suen, et. al. (Oct 2014) “Evolving Java-based APIs”<br /> <h2> Footnotes</h2> <ol class="my-list"> <li><a name="foot1"></a>We actually did have a hilarious bug in WebRTC where it was possible to set the volume to <a href="https://www.youtube.com/watch?v=KOO5S4vxi0o">1.1</a>, but only 0.0-1.0 was supposed to be allowed. No, really. Thus, our WebRTC implementation must be louder than the others since everybody knows 1.1 must be louder than 1.0.</li> </ol> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>Author: Patrik Höglund </i><br /> <br /> As we all know, software development is a complicated activity where we develop features and applications to provide value to our users. Furthermore, any nontrivial modern software is composed out of other software. For instance, the Chrome web browser pulls <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/&sq=package:chromium">roughly a hundred libraries</a> into its third_party folder when you build the browser. The most significant of these libraries is Blink, the rendering engine, but there’s also ffmpeg for image processing, skia for low-level 2D graphics, and WebRTC for real-time communication (to name a few). <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjD2L7vtCiPSjIHJ1CHBI-Uc1CGVq6_pSplgp47TUXYAZsYC7eHyl8SN9UnWr4ts06UKX7of3OYXWbyS9doxiyP5kgpbl7NBfw8TaXe3CEG8Mp2qCXBpWmPrQPlah6Dw4xIE1kK/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjD2L7vtCiPSjIHJ1CHBI-Uc1CGVq6_pSplgp47TUXYAZsYC7eHyl8SN9UnWr4ts06UKX7of3OYXWbyS9doxiyP5kgpbl7NBfw8TaXe3CEG8Mp2qCXBpWmPrQPlah6Dw4xIE1kK/s1600/image01.png" /></a></div> <div style="text-align: center;"> <i>Figure 1. Holy dependencies, Batman!</i></div> <br /> There are many reasons to use software libraries. Why write your own phone number parser when you can use <a href="https://github.com/googlei18n/libphonenumber">libphonenumber</a>, which is battle-tested by real use in Android and Chrome and available under a permissive license? Using such software frees you up to focus on the core of your software so you can deliver a unique experience to your users. On the other hand, you need to keep your application up to date with changes in the library (you want that latest bug fix, right?), and you also run a risk of such a change breaking your application. This article will examine that integration problem and how you can reduce the risks associated with it.<br /> <h2> Updating Dependencies is Hard</h2> The simplest solution is to check in a copy of the library, build with it, and avoid touching it as much as possible. This solution, however, can be problematic because you miss out on bug fixes and new features in the library. What if you need a new feature or bug fix that just made it in? You have a few options:<br /> <ul class="my-list"> <li>Update the library to its latest release. If it’s been a long time since you did this, it can be quite risky and you may have to spend significant testing resources to ensure all the accumulated changes don’t break your application. You may have to catch up to interface changes in the library as well. </li> <li>Cherry-pick the feature/bug fix you want into your copy of the library. This is even riskier because your cherry-picked patches may depend on other changes in the library in subtle ways. Also, you still are not up to date with the latest version. </li> <li>Find some way to make do without the feature or bug fix.</li> </ul> None of the above options are very good. Using this ad-hoc updating model can work if there’s a low volume of changes in the library and our requirements on the library don’t change very often. Even if that is the case, what will you do if a critical zero-day exploit is discovered in your socket library? <br /> <br /> One way to mitigate the update risk is to integrate more often with your dependencies. As an extreme example, let’s look at Chrome. <br /> <br /> In Chrome development, there’s a massive amount of change going into its dependencies. The Blink rendering engine lives in a separate code repository from the browser. Blink sees hundreds of code changes per day, and Chrome must integrate with Blink often since it’s an important part of the browser. Another example is the WebRTC implementation, where a large part of Chrome’s implementation resides in the webrtc.org repository. This article will focus on the latter because it’s the team I happen to work on.<br /> <h2> How “Rolling” Works </h2> The open-sourced WebRTC codebase is used by Chrome but also by a number of other companies working on WebRTC. Chrome uses a toolchain called depot_tools to manage dependencies, and there’s a checked-in text file called <a href="https://code.google.com/p/chromium/codesearch#chromium/src/DEPS&q=DEPS&sq=package:chromium&l=1">DEPS</a> where dependencies are managed. It looks roughly like this:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">{ <span style="color: #4f8f00;"># ... </span> <span style="color: #ff2600;">'src/third_party/webrtc'</span>: <span style="color: #ff2600;">'https://chromium.googlesource.com/'</span> + <span style="color: #ff2600;">'external/webrtc/trunk/webrtc.git'</span> + <span style="color: #ff2600;">'@'</span> + <span style="color: #ff2600;">'5727038f572c517204e1642b8bc69b25381c4e9f'</span>, } </pre> <br /> The above means we should pull WebRTC from the specified git repository at the <span style="font-family: Courier New, Courier, monospace;">572703...</span> hash, similar to other dependency-provisioning frameworks. To build Chrome with a new version, we change the hash and check in a new version of the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/DEPS&q=DEPS&sq=package:chromium&l=1">DEPS</a> file. If the library’s API has changed, we must update Chrome to use the new API in the same patch. This process is known as <i>rolling</i> WebRTC to a new version. <br /> <br /> Now the problem is that we have changed the code going into Chrome. Maybe <a href="https://code.google.com/p/chromium/issues/detail?id=430001">getUserMedia has started crashing on Android</a>, or maybe the browser no longer boots on Windows. We don’t know until we have built and run all the tests. Therefore a roll patch is subject to the same presubmit checks as any Chrome patch (i.e. <a href="https://www.chromium.org/developers/testing/try-server-usage">many tests, on all platforms we ship on</a>). However, roll patches can be considerably more painful and risky than other patches. <br /> <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTH4ArXq9A6nAaV-Sc_PNNafRrP3BL2GA3-BoYb0smhQwSUnk2xKPh2Nx0L6IPIi-O9gsljux6p_85Sr0Mww1jtUplbxOGRTN_Glad3gtpnm8r_HcMQ0FEJy3cJwZL7Fn6HzvK/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTH4ArXq9A6nAaV-Sc_PNNafRrP3BL2GA3-BoYb0smhQwSUnk2xKPh2Nx0L6IPIi-O9gsljux6p_85Sr0Mww1jtUplbxOGRTN_Glad3gtpnm8r_HcMQ0FEJy3cJwZL7Fn6HzvK/s1600/image00.png" /></a></div> <div style="text-align: center;"> <i>Figure 2. Life of a Roll Patch.</i></div> <br /> On the WebRTC team we found ourselves in an uncomfortable position a couple years back. Developers would make changes to the webrtc.org code and there was a fair amount of churn in the interface, which meant we would have to update Chrome to adapt to those changes. Also we frequently broke tests and WebRTC functionality in Chrome because semantic changes had unexpected consequences in Chrome. Since rolls were so risky and painful to make, they started to happen less often, which made things even worse. There could be two weeks between rolls, which meant Chrome was hit by a large number of changes in one patch.<br /> <h2> Bots That Can See the Future: “FYI Bots” </h2> We found a way to mitigate this which we called FYI (for your information) bots. A bot is Chrome lingo for a continuous build machine which builds Chrome and runs tests. <br /> <br /> All the existing Chrome bots at that point would build Chrome as specified in the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/DEPS&q=DEPS&sq=package:chromium&l=1">DEPS</a> file, which meant they would build the WebRTC version we had rolled to up to that point. FYI bots replace that pinned version with WebRTC HEAD, but otherwise build and run Chrome-level tests as usual. Therefore: <br /> <br /> <ul class="my-list"> <li>If all the FYI bots were green, we knew a roll most likely would go smoothly. </li> <li>If the bots didn’t compile, we knew we would have to adapt Chrome to an interface change in the next roll patch. </li> <li>If the bots were red, we knew we either had a bug in WebRTC or that Chrome would have to be adapted to some semantic change in WebRTC.</li> </ul> The <a href="https://build.chromium.org/p/chromium.webrtc.fyi/waterfall">FYI “waterfall”</a> (a set of bots that builds and runs tests) is a straight copy of the <a href="https://build.chromium.org/p/chromium.webrtc/waterfall">main waterfall</a>, which is expensive in resources. We could have cheated and just set up FYI bots for one platform (say, Linux), but the most expensive regressions are platform-specific, so we reckoned the extra machines and maintenance were worth it.<br /> <h2> Making Gradual Interface Changes </h2> This solution helped but wasn’t quite satisfactory. We initially had the policy that it was fine to break the FYI bots since we could not update Chrome to use a new interface until the new interface had actually been rolled into Chrome. This, however, often caused the FYI bots to be compile-broken for days. We quickly started to suffer from <a href="http://www.martinfowler.com/articles/continuousIntegration.html#FixBrokenBuildsImmediately">red blindness</a> <a href="#ref1">[1]</a> and had no idea if we would break tests on the roll, especially if an interface change was made early in the roll cycle. <br /> <br /> The solution was to move to a more careful update policy for the WebRTC API. For the more technically inclined, “careful” here means “following the <a href="https://wiki.eclipse.org/Evolving_Java-based_APIs#API_Prime_Directive">API prime directive</a>” <a href="#ref2">[2]</a>. Consider this example:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">class</span> WebRtcAmplifier { ... <span style="color: #0433ff;">int</span> SetOutputVolume(<span style="color: #0433ff;">float</span> volume); }</pre> <br /> Normally we would just change the method’s signature when we needed to:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">class</span> WebRtcAmplifier { ... <span style="color: #0433ff;">int</span> SetOutputVolume(<span style="color: #0433ff;">float</span> volume, <span style="color: #0433ff;">bool</span> allow_eleven<a href="#foot1"><sup>1</sup></a>); }</pre> <br /> … but this would compile-break Chome until it could be updated. So we started doing it like this instead:<br /> <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #0433ff;">class</span> WebRtcAmplifier { ... <span style="color: #0433ff;">int</span> SetOutputVolume(<span style="color: #0433ff;">float</span> volume); <span style="color: #0433ff;">int</span> SetOutputVolume2(<span style="color: #0433ff;">float</span> volume, <span style="color: #0433ff;">bool</span> allow_eleven); }</pre> <br /> Then we could:<br /> <ol class="my-list"> <li>Roll into Chrome </li> <li>Make Chrome use SetOutputVolume2 </li> <li>Update SetOutputVolume’s signature </li> <li>Roll again and make Chrome use SetOutputVolume </li> <li>Delete SetOutputVolume2</li> </ol> This approach requires several steps but we end up with the right interface and at no point do we break Chrome.<br /> <h2> Results</h2> When we implemented the above, we could fix problems as they came up rather than in big batches on each roll. We could institute the policy that the FYI bots should always be green, and that changes breaking them should be immediately rolled back. This made a huge difference. The team could work smoother and roll more often. This reduced our risk quite a bit, particularly when Chrome was about to cut a new version branch. Instead of doing panicked and risky rolls around a release, we could work out issues in good time and stay in control.<br /> <br /> Another benefit of FYI bots is more granular <a href="https://chromeperf.appspot.com/report?masters=ChromiumWebRTCFYI&bots=chromium-webrtc-trunk-tot-rel-linux&tests=browser_tests%2Fagc_energy_diff_with_agc&checked=all">performance tests</a>. Before the FYI bots, it would frequently happen that a bunch of metrics regressed. However, it’s not fun to find which of the 100 patches in the roll caused the regression! With the FYI bots, we can see precisely which WebRTC revision caused the problem.<br /> <h2> Future Work: Optimistic Auto-rolling</h2> The final step on this ladder (short of actually merging the repositories) is auto-rolling. The Blink team implemented this with their <a href="https://www.chromium.org/blink/blinkrollbot">ARB (AutoRollBot)</a>. The bot wakes up periodically and tries to do a roll patch. If it fails on the trybots, it waits and tries again later (perhaps the trybots failed because of a flake or other temporary error, or perhaps the error was real but has been fixed).<br /> <br /> To pull auto-rolling off, you are going to need very good tests. That goes for any roll patch (or any patch, really), but if you’re edging closer to a release and an unstoppable flood of code changes keep breaking you, you’re not in a good place.<br /> <br /> <hr /> <h2> References</h2> <a name="ref1"></a>[1] Martin Fowler (May 2006) “Continuous Integration”<br /> <a name="ref2"></a>[2] Dani Megert, Remy Chi Jian Suen, et. al. (Oct 2014) “Evolving Java-based APIs”<br /> <h2> Footnotes</h2> <ol class="my-list"> <li><a name="foot1"></a>We actually did have a hilarious bug in WebRTC where it was possible to set the volume to <a href="https://www.youtube.com/watch?v=KOO5S4vxi0o">1.1</a>, but only 0.0-1.0 was supposed to be allowed. No, really. Thus, our WebRTC implementation must be louder than the others since everybody knows 1.1 must be louder than 1.0.</li> </ol> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Multi-Repository Development&url=https://testing.googleblog.com/2015/05/multi-repository-development.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/05/multi-repository-development.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/05/multi-repository-development.html#comments' style='font-weight: 500; text-decoration: underline;'>6 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/05/multi-repository-development.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund' rel='tag'> Patrik Höglund </a> </span> </div> </div> </div> <div class='post' data-id='1113306492226112011' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html' itemprop='url' title='Just Say No to More End-to-End Tests'> Just Say No to More End-to-End Tests </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Wednesday, April 22, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by Mike Wacker </i><br /> <br /> At some point in your life, you can probably recall a movie that you and your friends all wanted to see, and that you and your friends all regretted watching afterwards. Or maybe you remember that time your team thought they’d found the next "killer feature" for their product, only to see that feature bomb after it was released. <br /> <br /> Good ideas often fail in practice, and in the world of testing, one pervasive good idea that often fails in practice is a testing strategy built around end-to-end tests. <br /> <br /> Testers can invest their time in writing many types of automated tests, including unit tests, integration tests, and end-to-end tests, but this strategy invests mostly in end-to-end tests that verify the product or service as a whole. Typically, these tests simulate real user scenarios.<br /> <h2> End-to-End Tests in Theory </h2> While relying primarily on end-to-end tests is a bad idea, one could certainly convince a reasonable person that the idea makes sense in theory. <br /> <br /> To start, number one on Google's list of <a href="//www.google.com/about/company/philosophy/">ten things we know to be true</a> is: "Focus on the user and all else will follow." Thus, end-to-end tests that focus on real user scenarios sound like a great idea. Additionally, this strategy broadly appeals to many constituencies:<br /> <ul class="my-list"> <li><b>Developers</b> like it because it offloads most, if not all, of the testing to others. </li> <li><b>Managers and decision-makers</b> like it because tests that simulate real user scenarios can help them easily determine how a failing test would impact the user. </li> <li><b>Testers</b> like it because they often worry about missing a bug or writing a test that does not verify real-world behavior; writing tests from the user's perspective often avoids both problems and gives the tester a greater sense of accomplishment. </li> </ul> <h2> End-to-End Tests in Practice </h2> So if this testing strategy sounds so good in theory, then where does it go wrong in practice? To demonstrate, I present the following composite sketch based on a collection of real experiences familiar to both myself and other testers. In this sketch, a team is building a service for editing documents online (e.g., Google Docs). <br /> <br /> Let's assume the team already has some fantastic test infrastructure in place. Every night:<br /> <ol class="my-list"> <li>The latest version of the service is built. </li> <li>This version is then deployed to the team's testing environment. </li> <li>All end-to-end tests then run against this testing environment. </li> <li>An email report summarizing the test results is sent to the team.</li> </ol> <br /> The deadline is approaching fast as our team codes new features for their next release. To maintain a high bar for product quality, they also require that at least 90% of their end-to-end tests pass before features are considered complete. Currently, that deadline is one day away: <br /> <br /> <table class="my-bordered-table" style="color: black;"> <tbody> <tr><td style="white-space: nowrap;">Days Left</td><td>Pass %</td><td>Notes</td></tr> <tr style="background-color: red;"><td>1</td><td>5%</td><td>Everything is broken! Signing in to the service is broken. Almost all tests sign in a user, so almost all tests failed.</td></tr> <tr style="background-color: red;"><td>0</td><td>4%</td><td>A partner team we rely on deployed a bad build to their testing environment yesterday.</td></tr> <tr style="background-color: orange;"><td>-1</td><td>54%</td><td>A dev broke the save scenario yesterday (or the day before?). Half the tests save a document at some point in time. Devs spent most of the day determining if it's a frontend bug or a backend bug.</td></tr> <tr style="background-color: orange;"><td>-2</td><td>54%</td><td>It's a frontend bug, devs spent half of today figuring out where.</td></tr> <tr style="background-color: orange;"><td>-3</td><td>54%</td><td>A bad fix was checked in yesterday. The mistake was pretty easy to spot, though, and a correct fix was checked in today.</td></tr> <tr style="background-color: red;"><td>-4</td><td>1%</td><td>Hardware failures occurred in the lab for our testing environment.</td></tr> <tr style="background-color: yellow;"><td>-5</td><td>84%</td><td>Many small bugs hiding behind the big bugs (e.g., sign-in broken, save broken). Still working on the small bugs.</td></tr> <tr style="background-color: yellow;"><td>-6</td><td>87%</td><td>We should be above 90%, but are not for some reason.</td></tr> <tr style="background-color: lime;"><td>-7</td><td>89.54%</td><td>(Rounds up to 90%, close enough.) No fixes were checked in yesterday, so the tests must have been flaky yesterday.</td></tr> </tbody></table> <br /> <h3> Analysis </h3> Despite numerous problems, the tests ultimately did catch real bugs.<br /> <br /> <h4> What Went Well </h4> <ul class="my-list"> <li>Customer-impacting bugs were identified and fixed before they reached the customer.</li> </ul> <br /> <h4> What Went Wrong </h4> <ul class="my-list"> <li>The team completed their coding milestone a week late (and worked a lot of overtime). </li> <li>Finding the root cause for a failing end-to-end test is painful and can take a long time. </li> <li>Partner failures and lab failures ruined the test results on multiple days. </li> <li>Many smaller bugs were hidden behind bigger bugs. </li> <li>End-to-end tests were flaky at times. </li> <li>Developers had to wait until the following day to know if a fix worked or not. </li> </ul> <br /> So now that we know what went wrong with the end-to-end strategy, we need to change our approach to testing to avoid many of these problems. But what is the right approach?<br /> <h2> The True Value of Tests </h2> Typically, a tester's job ends once they have a failing test. A bug is filed, and then it's the developer's job to fix the bug. To identify where the end-to-end strategy breaks down, however, we need to think outside this box and approach the problem from first principles. If we "focus on the user (and all else will follow)," we have to ask ourselves how a failing test benefits the user. Here is the answer:<br /> <br /> <b>A failing test does not directly benefit the user. </b><br /> <br /> While this statement seems shocking at first, it is true. If a product works, it works, whether a test says it works or not. If a product is broken, it is broken, whether a test says it is broken or not. So, if failing tests do not benefit the user, then what does benefit the user?<br /> <br /> <b>A bug fix directly benefits the user.</b><br /> <br /> The user will only be happy when that unintended behavior - the bug - goes away. Obviously, to fix a bug, you must know the bug exists. To know the bug exists, ideally you have a test that catches the bug (because the user will find the bug if the test does not). But in that entire process, from failing test to bug fix, value is only added at the very last step. <br /> <br /> <table class="my-bordered-table" style="color: black; text-align: left;"> <tbody> <tr><td>Stage</td><td style="background-color: red;">Failing Test</td><td style="background-color: red;">Bug Opened</td><td style="background-color: lime;">Bug Fixed</td></tr> <tr><td>Value Added</td><td style="background-color: red;">No</td><td style="background-color: red;">No</td><td style="background-color: lime;">Yes</td></tr> </tbody></table> <br /> Thus, to evaluate any testing strategy, you cannot just evaluate how it finds bugs. You also must evaluate how it enables developers to fix (and even prevent) bugs.<br /> <h2> Building the Right Feedback Loop</h2> Tests create a feedback loop that informs the developer whether the product is working or not. The ideal feedback loop has several properties:<br /> <ul class="my-list"> <li><b>It's fast</b>. No developer wants to wait hours or days to find out if their change works. Sometimes the change does not work - nobody is perfect - and the feedback loop needs to run multiple times. A faster feedback loop leads to faster fixes. If the loop is fast enough, developers may even run tests before checking in a change. </li> <li><b>It's reliable</b>. No developer wants to spend hours debugging a test, only to find out it was a flaky test. Flaky tests reduce the developer's trust in the test, and as a result flaky tests are often ignored, even when they find real product issues. </li> <li><b>It isolates failures</b>. To fix a bug, developers need to find the specific lines of code causing the bug. When a product contains millions of lines of codes, and the bug could be anywhere, it's like trying to find a needle in a haystack. </li> </ul> <h2> Think Smaller, Not Larger</h2> So how do we create that ideal feedback loop? By thinking smaller, not larger.<br /> <br /> <h3> Unit Tests</h3> Unit tests take a small piece of the product and test that piece in isolation. They tend to create that ideal feedback loop:<br /> <br /> <ul class="my-list"> <li><b>Unit tests are fast</b>. We only need to build a small unit to test it, and the tests also tend to be rather small. In fact, one tenth of a second is considered slow for unit tests. </li> <li><b>Unit tests are reliable</b>. Simple systems and small units in general tend to suffer much less from flakiness. Furthermore, best practices for unit testing - in particular practices related to hermetic tests - will remove flakiness entirely. </li> <li><b>Unit tests isolate failures</b>. Even if a product contains millions of lines of code, if a unit test fails, you only need to search that small unit under test to find the bug. </li> </ul> <br /> Writing effective unit tests requires skills in areas such as dependency management, mocking, and hermetic testing. I won't cover these skills here, but as a start, the typical example offered to new Googlers (or Nooglers) is how Google <a href="https://github.com/google/guava/blob/master/guava/src/com/google/common/base/Stopwatch.java">builds</a> and <a href="https://github.com/google/guava/blob/master/guava-tests/test/com/google/common/base/StopwatchTest.java">tests</a> a stopwatch.<br /> <br /> <h3> Unit Tests vs. End-to-End Tests</h3> With end-to-end tests, you have to wait: first for the entire product to be built, then for it to be deployed, and finally for all end-to-end tests to run. When the tests do run, flaky tests tend to be a fact of life. And even if a test finds a bug, that bug could be anywhere in the product.<br /> <br /> Although end-to-end tests do a better job of simulating real user scenarios, this advantage quickly becomes outweighed by all the disadvantages of the end-to-end feedback loop: <br /> <br /> <table class="my-bordered-table" style="color: black;"> <tbody> <tr><td></td><td>Unit</td><td>End-toEnd</td></tr> <tr><td>Fast</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <br /></td></tr> <tr><td>Reliable</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <div class="separator" style="clear: both; text-align: center;"> <br /></div> <br /></td></tr> <tr><td>Isolates Failures</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <br /></td></tr> <tr><td>Simulates a Real User</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td></tr> </tbody></table> <br /> <h3> Integration Tests</h3> Unit tests do have one major disadvantage: even if the units work well in isolation, you do not know if they work well together. But even then, you do not necessarily need end-to-end tests. For that, you can use an integration test. An integration test takes a small group of units, often two units, and tests their behavior as a whole, verifying that they coherently work together.<br /> <br /> If two units do not integrate properly, why write an end-to-end test when you can write a much smaller, more focused integration test that will detect the same bug? While you do need to think larger, you only need to think a little larger to verify that units work together.<br /> <h2> Testing Pyramid</h2> Even with both unit tests and integration tests, you probably still will want a small number of end-to-end tests to verify the system as a whole. To find the right balance between all three test types, the best visual aid to use is the testing pyramid. Here is a simplified version of the <a href="https://docs.google.com/presentation/d/15gNk21rjer3xo-b1ZqyQVGebOp_aPvHU3YH7YnOMxtE/edit#slide=id.g437663ce1_53_98">testing pyramid</a> from the opening keynote of the <a href="https://developers.google.com/google-test-automation-conference/2014/">2014 Google Test Automation Conference</a>: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj51PqHi78kut8oYzxpBI0w_GDGGYsFaVDIIPWpBmT0tCDAssoPogyViYE9G8kTRSfT2wGdKsdFaokO8iBHjFYd5Js33hp2OT04LXfLQK9mNPPU0fURBbpno9FIwV7PbIwzL2X4/s1600/image02.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="276" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj51PqHi78kut8oYzxpBI0w_GDGGYsFaVDIIPWpBmT0tCDAssoPogyViYE9G8kTRSfT2wGdKsdFaokO8iBHjFYd5Js33hp2OT04LXfLQK9mNPPU0fURBbpno9FIwV7PbIwzL2X4/s1600/image02.png" width="320" /></a></div> <br /> <br /> The bulk of your tests are unit tests at the bottom of the pyramid. As you move up the pyramid, your tests gets larger, but at the same time the number of tests (the width of your pyramid) gets smaller.<br /> <br /> As a good first guess, Google often suggests a 70/20/10 split: 70% unit tests, 20% integration tests, and 10% end-to-end tests. The exact mix will be different for each team, but in general, it should retain that pyramid shape. Try to avoid these anti-patterns:<br /> <ul class="my-list"> <li><b>Inverted pyramid/ice cream cone</b>. The team relies primarily on end-to-end tests, using few integration tests and even fewer unit tests. </li> <li><b>Hourglass</b>. The team starts with a lot of unit tests, then uses end-to-end tests where integration tests should be used. The hourglass has many unit tests at the bottom and many end-to-end tests at the top, but few integration tests in the middle. </li> </ul> Just like a regular pyramid tends to be the most stable structure in real life, the testing pyramid also tends to be the most stable testing strategy. <br /> <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by Mike Wacker </i><br /> <br /> At some point in your life, you can probably recall a movie that you and your friends all wanted to see, and that you and your friends all regretted watching afterwards. Or maybe you remember that time your team thought they’d found the next "killer feature" for their product, only to see that feature bomb after it was released. <br /> <br /> Good ideas often fail in practice, and in the world of testing, one pervasive good idea that often fails in practice is a testing strategy built around end-to-end tests. <br /> <br /> Testers can invest their time in writing many types of automated tests, including unit tests, integration tests, and end-to-end tests, but this strategy invests mostly in end-to-end tests that verify the product or service as a whole. Typically, these tests simulate real user scenarios.<br /> <h2> End-to-End Tests in Theory </h2> While relying primarily on end-to-end tests is a bad idea, one could certainly convince a reasonable person that the idea makes sense in theory. <br /> <br /> To start, number one on Google's list of <a href="//www.google.com/about/company/philosophy/">ten things we know to be true</a> is: "Focus on the user and all else will follow." Thus, end-to-end tests that focus on real user scenarios sound like a great idea. Additionally, this strategy broadly appeals to many constituencies:<br /> <ul class="my-list"> <li><b>Developers</b> like it because it offloads most, if not all, of the testing to others. </li> <li><b>Managers and decision-makers</b> like it because tests that simulate real user scenarios can help them easily determine how a failing test would impact the user. </li> <li><b>Testers</b> like it because they often worry about missing a bug or writing a test that does not verify real-world behavior; writing tests from the user's perspective often avoids both problems and gives the tester a greater sense of accomplishment. </li> </ul> <h2> End-to-End Tests in Practice </h2> So if this testing strategy sounds so good in theory, then where does it go wrong in practice? To demonstrate, I present the following composite sketch based on a collection of real experiences familiar to both myself and other testers. In this sketch, a team is building a service for editing documents online (e.g., Google Docs). <br /> <br /> Let's assume the team already has some fantastic test infrastructure in place. Every night:<br /> <ol class="my-list"> <li>The latest version of the service is built. </li> <li>This version is then deployed to the team's testing environment. </li> <li>All end-to-end tests then run against this testing environment. </li> <li>An email report summarizing the test results is sent to the team.</li> </ol> <br /> The deadline is approaching fast as our team codes new features for their next release. To maintain a high bar for product quality, they also require that at least 90% of their end-to-end tests pass before features are considered complete. Currently, that deadline is one day away: <br /> <br /> <table class="my-bordered-table" style="color: black;"> <tbody> <tr><td style="white-space: nowrap;">Days Left</td><td>Pass %</td><td>Notes</td></tr> <tr style="background-color: red;"><td>1</td><td>5%</td><td>Everything is broken! Signing in to the service is broken. Almost all tests sign in a user, so almost all tests failed.</td></tr> <tr style="background-color: red;"><td>0</td><td>4%</td><td>A partner team we rely on deployed a bad build to their testing environment yesterday.</td></tr> <tr style="background-color: orange;"><td>-1</td><td>54%</td><td>A dev broke the save scenario yesterday (or the day before?). Half the tests save a document at some point in time. Devs spent most of the day determining if it's a frontend bug or a backend bug.</td></tr> <tr style="background-color: orange;"><td>-2</td><td>54%</td><td>It's a frontend bug, devs spent half of today figuring out where.</td></tr> <tr style="background-color: orange;"><td>-3</td><td>54%</td><td>A bad fix was checked in yesterday. The mistake was pretty easy to spot, though, and a correct fix was checked in today.</td></tr> <tr style="background-color: red;"><td>-4</td><td>1%</td><td>Hardware failures occurred in the lab for our testing environment.</td></tr> <tr style="background-color: yellow;"><td>-5</td><td>84%</td><td>Many small bugs hiding behind the big bugs (e.g., sign-in broken, save broken). Still working on the small bugs.</td></tr> <tr style="background-color: yellow;"><td>-6</td><td>87%</td><td>We should be above 90%, but are not for some reason.</td></tr> <tr style="background-color: lime;"><td>-7</td><td>89.54%</td><td>(Rounds up to 90%, close enough.) No fixes were checked in yesterday, so the tests must have been flaky yesterday.</td></tr> </tbody></table> <br /> <h3> Analysis </h3> Despite numerous problems, the tests ultimately did catch real bugs.<br /> <br /> <h4> What Went Well </h4> <ul class="my-list"> <li>Customer-impacting bugs were identified and fixed before they reached the customer.</li> </ul> <br /> <h4> What Went Wrong </h4> <ul class="my-list"> <li>The team completed their coding milestone a week late (and worked a lot of overtime). </li> <li>Finding the root cause for a failing end-to-end test is painful and can take a long time. </li> <li>Partner failures and lab failures ruined the test results on multiple days. </li> <li>Many smaller bugs were hidden behind bigger bugs. </li> <li>End-to-end tests were flaky at times. </li> <li>Developers had to wait until the following day to know if a fix worked or not. </li> </ul> <br /> So now that we know what went wrong with the end-to-end strategy, we need to change our approach to testing to avoid many of these problems. But what is the right approach?<br /> <h2> The True Value of Tests </h2> Typically, a tester's job ends once they have a failing test. A bug is filed, and then it's the developer's job to fix the bug. To identify where the end-to-end strategy breaks down, however, we need to think outside this box and approach the problem from first principles. If we "focus on the user (and all else will follow)," we have to ask ourselves how a failing test benefits the user. Here is the answer:<br /> <br /> <b>A failing test does not directly benefit the user. </b><br /> <br /> While this statement seems shocking at first, it is true. If a product works, it works, whether a test says it works or not. If a product is broken, it is broken, whether a test says it is broken or not. So, if failing tests do not benefit the user, then what does benefit the user?<br /> <br /> <b>A bug fix directly benefits the user.</b><br /> <br /> The user will only be happy when that unintended behavior - the bug - goes away. Obviously, to fix a bug, you must know the bug exists. To know the bug exists, ideally you have a test that catches the bug (because the user will find the bug if the test does not). But in that entire process, from failing test to bug fix, value is only added at the very last step. <br /> <br /> <table class="my-bordered-table" style="color: black; text-align: left;"> <tbody> <tr><td>Stage</td><td style="background-color: red;">Failing Test</td><td style="background-color: red;">Bug Opened</td><td style="background-color: lime;">Bug Fixed</td></tr> <tr><td>Value Added</td><td style="background-color: red;">No</td><td style="background-color: red;">No</td><td style="background-color: lime;">Yes</td></tr> </tbody></table> <br /> Thus, to evaluate any testing strategy, you cannot just evaluate how it finds bugs. You also must evaluate how it enables developers to fix (and even prevent) bugs.<br /> <h2> Building the Right Feedback Loop</h2> Tests create a feedback loop that informs the developer whether the product is working or not. The ideal feedback loop has several properties:<br /> <ul class="my-list"> <li><b>It's fast</b>. No developer wants to wait hours or days to find out if their change works. Sometimes the change does not work - nobody is perfect - and the feedback loop needs to run multiple times. A faster feedback loop leads to faster fixes. If the loop is fast enough, developers may even run tests before checking in a change. </li> <li><b>It's reliable</b>. No developer wants to spend hours debugging a test, only to find out it was a flaky test. Flaky tests reduce the developer's trust in the test, and as a result flaky tests are often ignored, even when they find real product issues. </li> <li><b>It isolates failures</b>. To fix a bug, developers need to find the specific lines of code causing the bug. When a product contains millions of lines of codes, and the bug could be anywhere, it's like trying to find a needle in a haystack. </li> </ul> <h2> Think Smaller, Not Larger</h2> So how do we create that ideal feedback loop? By thinking smaller, not larger.<br /> <br /> <h3> Unit Tests</h3> Unit tests take a small piece of the product and test that piece in isolation. They tend to create that ideal feedback loop:<br /> <br /> <ul class="my-list"> <li><b>Unit tests are fast</b>. We only need to build a small unit to test it, and the tests also tend to be rather small. In fact, one tenth of a second is considered slow for unit tests. </li> <li><b>Unit tests are reliable</b>. Simple systems and small units in general tend to suffer much less from flakiness. Furthermore, best practices for unit testing - in particular practices related to hermetic tests - will remove flakiness entirely. </li> <li><b>Unit tests isolate failures</b>. Even if a product contains millions of lines of code, if a unit test fails, you only need to search that small unit under test to find the bug. </li> </ul> <br /> Writing effective unit tests requires skills in areas such as dependency management, mocking, and hermetic testing. I won't cover these skills here, but as a start, the typical example offered to new Googlers (or Nooglers) is how Google <a href="https://github.com/google/guava/blob/master/guava/src/com/google/common/base/Stopwatch.java">builds</a> and <a href="https://github.com/google/guava/blob/master/guava-tests/test/com/google/common/base/StopwatchTest.java">tests</a> a stopwatch.<br /> <br /> <h3> Unit Tests vs. End-to-End Tests</h3> With end-to-end tests, you have to wait: first for the entire product to be built, then for it to be deployed, and finally for all end-to-end tests to run. When the tests do run, flaky tests tend to be a fact of life. And even if a test finds a bug, that bug could be anywhere in the product.<br /> <br /> Although end-to-end tests do a better job of simulating real user scenarios, this advantage quickly becomes outweighed by all the disadvantages of the end-to-end feedback loop: <br /> <br /> <table class="my-bordered-table" style="color: black;"> <tbody> <tr><td></td><td>Unit</td><td>End-toEnd</td></tr> <tr><td>Fast</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <br /></td></tr> <tr><td>Reliable</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <div class="separator" style="clear: both; text-align: center;"> <br /></div> <br /></td></tr> <tr><td>Isolates Failures</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <br /></td></tr> <tr><td>Simulates a Real User</td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgnADvsltmYlL_hIC0Deu0-pwvy9xpCP-CQ-OlFyjNFo8eDNF1ocdqR6azPhikTSBzoqYPdDaA4Y4kZeYFVyOWJjjvNfH7jX6g1iYmuc78CkmZSc8SimX7qvTnL9G0V3NY-uhyphenhyphen9/s1600/sad.png" /></a></div> <br /></td><td><div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjSnZKU-GvPv3dV0aMKnqAnqUebK7IFhnO-O11iM9Imwjidrk3JsTx_K8rF0LwPh9Hd_OiRjLVkngEOtt-oY2ZvWrOTM8LH5SgyrO7NWk4ruA1e4lVwQ4xPrzH4N4485u8WkkSI/s1600/happy.png" /></a></div> <br /></td></tr> </tbody></table> <br /> <h3> Integration Tests</h3> Unit tests do have one major disadvantage: even if the units work well in isolation, you do not know if they work well together. But even then, you do not necessarily need end-to-end tests. For that, you can use an integration test. An integration test takes a small group of units, often two units, and tests their behavior as a whole, verifying that they coherently work together.<br /> <br /> If two units do not integrate properly, why write an end-to-end test when you can write a much smaller, more focused integration test that will detect the same bug? While you do need to think larger, you only need to think a little larger to verify that units work together.<br /> <h2> Testing Pyramid</h2> Even with both unit tests and integration tests, you probably still will want a small number of end-to-end tests to verify the system as a whole. To find the right balance between all three test types, the best visual aid to use is the testing pyramid. Here is a simplified version of the <a href="https://docs.google.com/presentation/d/15gNk21rjer3xo-b1ZqyQVGebOp_aPvHU3YH7YnOMxtE/edit#slide=id.g437663ce1_53_98">testing pyramid</a> from the opening keynote of the <a href="https://developers.google.com/google-test-automation-conference/2014/">2014 Google Test Automation Conference</a>: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj51PqHi78kut8oYzxpBI0w_GDGGYsFaVDIIPWpBmT0tCDAssoPogyViYE9G8kTRSfT2wGdKsdFaokO8iBHjFYd5Js33hp2OT04LXfLQK9mNPPU0fURBbpno9FIwV7PbIwzL2X4/s1600/image02.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="276" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj51PqHi78kut8oYzxpBI0w_GDGGYsFaVDIIPWpBmT0tCDAssoPogyViYE9G8kTRSfT2wGdKsdFaokO8iBHjFYd5Js33hp2OT04LXfLQK9mNPPU0fURBbpno9FIwV7PbIwzL2X4/s1600/image02.png" width="320" /></a></div> <br /> <br /> The bulk of your tests are unit tests at the bottom of the pyramid. As you move up the pyramid, your tests gets larger, but at the same time the number of tests (the width of your pyramid) gets smaller.<br /> <br /> As a good first guess, Google often suggests a 70/20/10 split: 70% unit tests, 20% integration tests, and 10% end-to-end tests. The exact mix will be different for each team, but in general, it should retain that pyramid shape. Try to avoid these anti-patterns:<br /> <ul class="my-list"> <li><b>Inverted pyramid/ice cream cone</b>. The team relies primarily on end-to-end tests, using few integration tests and even fewer unit tests. </li> <li><b>Hourglass</b>. The team starts with a lot of unit tests, then uses end-to-end tests where integration tests should be used. The hourglass has many unit tests at the bottom and many end-to-end tests at the top, but few integration tests in the middle. </li> </ul> Just like a regular pyramid tends to be the most stable structure in real life, the testing pyramid also tends to be the most stable testing strategy. <br /> <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Just Say No to More End-to-End Tests&url=https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html#comments' style='font-weight: 500; text-decoration: underline;'>84 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Mike%20Wacker' rel='tag'> Mike Wacker </a> </span> </div> </div> </div> <div class='post' data-id='3536158978043885342' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/04/quantum-quality.html' itemprop='url' title='Quantum Quality'> Quantum Quality </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Wednesday, April 01, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <div style="text-align: center;"> <span style="color: red;">UPDATE</span>: Hey, this was an April fool's joke but in fact we wished we could</div> <div style="text-align: center;"> have realized this idea and we are looking forward to the day this has</div> <div style="text-align: center;"> been worked out and becomes a reality.</div> <i><br /></i> <i>by Kevin Graney </i><br /> <br /> Here at Google we have a long history of capitalizing on the latest research and technology to improve the quality of our software. Over our past 16+ years as a company, what started with some humble unit tests has grown into a massive operation. As our software complexity increased, ever larger and more complex tests were dreamed up by our Software Engineers in Test (SETs). <br /> <br /> What we have come to realize is that our love of testing is a double-edged sword. On the one hand, large-scale testing keeps us honest and gives us confidence. It ensures our products remain reliable, our users' data is kept safe, and our engineers are able to work productively without fear of breaking things. On the other hand, it's expensive in both engineer and machine time. Our SETs have been working tirelessly to reduce the expense and latency of software tests at Google, while continuing to increase their quality. <br /> <br /> Today, we're excited to reveal how Google is tackling this challenge. In collaboration with the <a href="https://plus.google.com/+QuantumAILab">Quantum AI Lab</a>, SETs at Google have been busy revolutionizing how software is tested. The theory is relatively simple: bits in a traditional computer are either zero or one, but bits in a quantum computer can be both one and zero at the same time. This is known as superposition, and the classic example is <a href="http://en.wikipedia.org/wiki/Schr%C3%B6dinger%27s_cat">Schrodinger's cat</a>. Through some clever math and cutting edge electrical engineering, researchers at Google have figured out how to utilize superposition to vastly improve the quality of our software testing and the speed at which our tests run. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEge2vl_almMEm0kZnoPzyNmbkRmmliMn92Ikdz_2aZP32Li2JNlTQiCjNawthtPUBY2oPlScJmG1anLNDy0YBViIb06ExbzhF2w8Z-k8TDWuDhUcUJmEWKxYsSwZZ-K8AtPJ0Vx/s1600/image04.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="316" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEge2vl_almMEm0kZnoPzyNmbkRmmliMn92Ikdz_2aZP32Li2JNlTQiCjNawthtPUBY2oPlScJmG1anLNDy0YBViIb06ExbzhF2w8Z-k8TDWuDhUcUJmEWKxYsSwZZ-K8AtPJ0Vx/s1600/image04.png" width="400" /></a></div> <br /> <div style="text-align: center;"> <b>Figure 1</b> Some qubits inside a Google quantum device. </div> <br /> With superposition, tests at Google are now able to simultaneously model every possible state of the application under test. The state of the application can be thought of as an <i>n</i> bit sequential memory buffer, consistent with the traditional <a href="http://en.wikipedia.org/wiki/Von_Neumann_architecture">Von Neuman architecture</a> of computing. Because each bit under superposition is simultaneously a 0 and a 1, these tests can simulate 2<span style="font-size: small; vertical-align: super;"><i>n</i></span> different application states at any given instant in time in <i>O(n)</i> space. Each of these application states can be mutated by application logic to another state in constant time using quantum algorithms developed by Google researchers. These two properties together allow us to build a state transition graph of the application under test that shows every possible application state and all possible transitions to other application states. Using traditional computing methods this problem has intractable time complexity, but after leveraging superposition and our quantum algorithms it becomes relatively fast and cheap. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5HHQOzaFe_cvgMhC9BYKytWlsRIm8SUOUu7K-5IyJ9ujyAb_Z3JSJhLq0TxrET5cYVXgZQkV9mQ3vehGFQo156jboghnbI6-m6lamlbg7_RMh-sPahUpGlLIeUdDEQRPkoovg/s1600/state.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5HHQOzaFe_cvgMhC9BYKytWlsRIm8SUOUu7K-5IyJ9ujyAb_Z3JSJhLq0TxrET5cYVXgZQkV9mQ3vehGFQo156jboghnbI6-m6lamlbg7_RMh-sPahUpGlLIeUdDEQRPkoovg/s1600/state.png" /></a></div> <div style="margin-left: auto; margin-right: auto; width: 500px;"> <b>Figure 2</b> The application state graph for a demonstrative 3-bit application. If the start state is 001 then 000, 110, 111, and 011 are all unreachable states. States 010 and 100 both result in deadlock. </div> <br /> <div style="text-align: justify;"> Once we have the state transition graph for the application under test, testing it becomes almost trivial. Given the initial startup state of the application, i.e. the executable bits of the application stored on disk, we can find from the application's state transition graph all reachable states. Assertions that ensure proper behavior are then written against the reachable subset of the transition graph. This paradigm of test writing allows both Google's security engineers and software engineers to work more productively. A security engineer can write a test, for example, that asserts "no executable memory regions become mutated in any reachable state". This one test effectively eliminates the potential for security flaws that result from <a href="http://en.wikipedia.org/wiki/Memory_safety">memory safety violations</a>. A test engineer can write higher level assertions using graph traversal methods that ensure data integrity is maintained across a subset of application state transitions. Tests of this nature can detect data corruption bugs. </div> <br /> We're excited about the work our team has done so far to push the envelope in the field of quantum software quality. We're just getting started, but based on early dogfood results among Googlers we believe the potential of this work is huge. Stay tuned! <br /> <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <div style="text-align: center;"> <span style="color: red;">UPDATE</span>: Hey, this was an April fool's joke but in fact we wished we could</div> <div style="text-align: center;"> have realized this idea and we are looking forward to the day this has</div> <div style="text-align: center;"> been worked out and becomes a reality.</div> <i><br /></i> <i>by Kevin Graney </i><br /> <br /> Here at Google we have a long history of capitalizing on the latest research and technology to improve the quality of our software. Over our past 16+ years as a company, what started with some humble unit tests has grown into a massive operation. As our software complexity increased, ever larger and more complex tests were dreamed up by our Software Engineers in Test (SETs). <br /> <br /> What we have come to realize is that our love of testing is a double-edged sword. On the one hand, large-scale testing keeps us honest and gives us confidence. It ensures our products remain reliable, our users' data is kept safe, and our engineers are able to work productively without fear of breaking things. On the other hand, it's expensive in both engineer and machine time. Our SETs have been working tirelessly to reduce the expense and latency of software tests at Google, while continuing to increase their quality. <br /> <br /> Today, we're excited to reveal how Google is tackling this challenge. In collaboration with the <a href="https://plus.google.com/+QuantumAILab">Quantum AI Lab</a>, SETs at Google have been busy revolutionizing how software is tested. The theory is relatively simple: bits in a traditional computer are either zero or one, but bits in a quantum computer can be both one and zero at the same time. This is known as superposition, and the classic example is <a href="http://en.wikipedia.org/wiki/Schr%C3%B6dinger%27s_cat">Schrodinger's cat</a>. Through some clever math and cutting edge electrical engineering, researchers at Google have figured out how to utilize superposition to vastly improve the quality of our software testing and the speed at which our tests run. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEge2vl_almMEm0kZnoPzyNmbkRmmliMn92Ikdz_2aZP32Li2JNlTQiCjNawthtPUBY2oPlScJmG1anLNDy0YBViIb06ExbzhF2w8Z-k8TDWuDhUcUJmEWKxYsSwZZ-K8AtPJ0Vx/s1600/image04.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="316" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEge2vl_almMEm0kZnoPzyNmbkRmmliMn92Ikdz_2aZP32Li2JNlTQiCjNawthtPUBY2oPlScJmG1anLNDy0YBViIb06ExbzhF2w8Z-k8TDWuDhUcUJmEWKxYsSwZZ-K8AtPJ0Vx/s1600/image04.png" width="400" /></a></div> <br /> <div style="text-align: center;"> <b>Figure 1</b> Some qubits inside a Google quantum device. </div> <br /> With superposition, tests at Google are now able to simultaneously model every possible state of the application under test. The state of the application can be thought of as an <i>n</i> bit sequential memory buffer, consistent with the traditional <a href="http://en.wikipedia.org/wiki/Von_Neumann_architecture">Von Neuman architecture</a> of computing. Because each bit under superposition is simultaneously a 0 and a 1, these tests can simulate 2<span style="font-size: small; vertical-align: super;"><i>n</i></span> different application states at any given instant in time in <i>O(n)</i> space. Each of these application states can be mutated by application logic to another state in constant time using quantum algorithms developed by Google researchers. These two properties together allow us to build a state transition graph of the application under test that shows every possible application state and all possible transitions to other application states. Using traditional computing methods this problem has intractable time complexity, but after leveraging superposition and our quantum algorithms it becomes relatively fast and cheap. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5HHQOzaFe_cvgMhC9BYKytWlsRIm8SUOUu7K-5IyJ9ujyAb_Z3JSJhLq0TxrET5cYVXgZQkV9mQ3vehGFQo156jboghnbI6-m6lamlbg7_RMh-sPahUpGlLIeUdDEQRPkoovg/s1600/state.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi5HHQOzaFe_cvgMhC9BYKytWlsRIm8SUOUu7K-5IyJ9ujyAb_Z3JSJhLq0TxrET5cYVXgZQkV9mQ3vehGFQo156jboghnbI6-m6lamlbg7_RMh-sPahUpGlLIeUdDEQRPkoovg/s1600/state.png" /></a></div> <div style="margin-left: auto; margin-right: auto; width: 500px;"> <b>Figure 2</b> The application state graph for a demonstrative 3-bit application. If the start state is 001 then 000, 110, 111, and 011 are all unreachable states. States 010 and 100 both result in deadlock. </div> <br /> <div style="text-align: justify;"> Once we have the state transition graph for the application under test, testing it becomes almost trivial. Given the initial startup state of the application, i.e. the executable bits of the application stored on disk, we can find from the application's state transition graph all reachable states. Assertions that ensure proper behavior are then written against the reachable subset of the transition graph. This paradigm of test writing allows both Google's security engineers and software engineers to work more productively. A security engineer can write a test, for example, that asserts "no executable memory regions become mutated in any reachable state". This one test effectively eliminates the potential for security flaws that result from <a href="http://en.wikipedia.org/wiki/Memory_safety">memory safety violations</a>. A test engineer can write higher level assertions using graph traversal methods that ensure data integrity is maintained across a subset of application state transitions. Tests of this nature can detect data corruption bugs. </div> <br /> We're excited about the work our team has done so far to push the envelope in the field of quantum software quality. We're just getting started, but based on early dogfood results among Googlers we believe the potential of this work is huge. Stay tuned! <br /> <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Quantum Quality&url=https://testing.googleblog.com/2015/04/quantum-quality.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/04/quantum-quality.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/04/quantum-quality.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/04/quantum-quality.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/April%20Fools' rel='tag'> April Fools </a> , <a class='label' href='https://testing.googleblog.com/search/label/Kevin%20Graney' rel='tag'> Kevin Graney </a> </span> </div> </div> </div> <div class='post' data-id='5953904353146570352' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/03/android-ui-automated-testing.html' itemprop='url' title='Android UI Automated Testing'> Android UI Automated Testing </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Friday, March 20, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by Mona El Mahdy </i><br /> <br /> <span style="font-size: large;"><b>Overview </b></span><br /> <br /> This post reviews four strategies for Android UI testing with the goal of creating UI tests that are fast, reliable, and easy to debug.<br /> <br /> Before we begin, let’s not forget an important rule: whatever can be unit tested should be unit tested. <a href="http://robolectric.org/">Robolectric</a> and <a href="http://tools.android.com/tech-docs/unit-testing-support">gradle unit tests support</a> are great examples of unit test frameworks for Android. UI tests, on the other hand, are used to verify that your application returns the correct UI output in response to a sequence of user actions on a device. <a href="http://developer.android.com/tools/testing-support-library/index.html#Espresso">Espresso</a> is a great framework for running UI actions and verifications in the same process. For more details on the Espresso and UI Automator tools, please see: <a href="http://developer.android.com/tools/testing-support-library/index.html">test support libraries</a>. <br /> <br /> The Google+ team has performed many iterations of UI testing. Below we discuss the lessons learned during each strategy of UI testing. Stay tuned for more posts with more details and code samples. <br /> <br /> <span style="font-size: large;"><b>Strategy 1: Using an End-To-End Test as a UI Test </b></span><br /> <br /> Let’s start with some definitions. A <b>UI test</b> ensures that your application returns the correct UI output in response to a sequence of user actions on a device. An <b>end-to-end (E2E)</b> test brings up the full system of your app including all backend servers and client app. E2E tests will guarantee that data is sent to the client app and that the entire system functions correctly. <br /> <br /> Usually, in order to make the application UI functional, you need data from backend servers, so UI tests need to simulate the data but not necessarily the backend servers. In many cases UI tests are confused with E2E tests because E2E is very similar to manual test scenarios. However, debugging and stabilizing E2E tests is very difficult due to many variables like network flakiness, authentication against real servers, size of your system, etc. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi0PUfVmtCBE3QE8Hi3L7OWM7i5iYLXbqxEalx1R4Q34C788Nk8C9wvWU33qWh6GA6FNJE7guyYHX6Tq3Tfx37uQTZU7CJ5WgBTjh_IECQGWKE2QtoChE7ZHUH_WoLlYBtX45Y2/s1600/image04.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="370" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi0PUfVmtCBE3QE8Hi3L7OWM7i5iYLXbqxEalx1R4Q34C788Nk8C9wvWU33qWh6GA6FNJE7guyYHX6Tq3Tfx37uQTZU7CJ5WgBTjh_IECQGWKE2QtoChE7ZHUH_WoLlYBtX45Y2/s1600/image04.png" width="400" /></a></div> <br /> When you use UI tests as E2E tests, you face the following problems:<br /> <ul style="line-height: 1.3em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Very large and slow tests. </li> <li>High flakiness rate due to timeouts and memory issues. </li> <li>Hard to debug/investigate failures. </li> <li>Authentication issues (ex: authentication from automated tests is very tricky).</li> </ul> <br /> Let’s see how these problems can be fixed using the following strategies.<br /> <br /> <span style="font-size: large;"><b>Strategy 2: Hermetic UI Testing using Fake Servers</b></span><br /> <br /> In this strategy, you avoid network calls and external dependencies, but you need to provide your application with data that drives the UI. Update your application to communicate to a local server rather than external one, and create a fake local server that provides data to your application. You then need a mechanism to generate the data needed by your application. This can be done using various approaches depending on your system design. One approach is to record server responses and replay them in your fake server. <br /> <br /> Once you have hermetic UI tests talking to a local fake server, you should also have <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">server hermetic tests</a>. This way you split your E2E test into a server side test, a client side test, and an integration test to verify that the server and client are in sync (for more details on integration tests, see the backend testing section of <a href="http://googletesting.blogspot.com/2013/08/how-google-team-tests-mobile-apps.html">blog</a>). <br /> <br /> Now, the client test flow looks like: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjmF32nIE84J1wkwAQvjjvvFg-hujwvIC7a922c68jdX7I56pTEbuDvhALwA7U1_rxpVSDLS30EdQZRVtW2JlN5DCSOxl6tWFzLIoT4qhjOE-44OooYUM6NatMO3HxDY8ky4Njf/s1600/image02.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjmF32nIE84J1wkwAQvjjvvFg-hujwvIC7a922c68jdX7I56pTEbuDvhALwA7U1_rxpVSDLS30EdQZRVtW2JlN5DCSOxl6tWFzLIoT4qhjOE-44OooYUM6NatMO3HxDY8ky4Njf/s1600/image02.png" width="201" /></a></div> <br /> While this approach drastically reduces the test size and flakiness rate, you still need to maintain a separate fake server as well as your test. Debugging is still not easy as you have two moving parts: the test and the local server. While test stability will be largely improved by this approach, the local server will cause some flakes. <br /> <br /> Let’s see how this could this be improved... <br /> <br /> <span style="font-size: large;"><b>Strategy 3: Dependency Injection Design for Apps. </b></span><br /> <br /> To remove the additional dependency of a fake server running on Android, you should use dependency injection in your application for swapping real module implementations with fake ones. One example is <a href="http://square.github.io/dagger/">Dagger</a>, or you can create your own dependency injection mechanism if needed. <br /> <br /> This will improve the testability of your app for both unit testing and UI testing, providing your tests with the ability to mock dependencies. In <a href="http://developer.android.com/tools/testing/testing_android.html">instrumentation testing</a>, the test apk and the app under test are loaded in the same process, so the test code has runtime access to the app code. Not only that, but you can also use classpath override (the fact that test classpath takes priority over app under test) to override a certain class and inject test fakes there. For example, To make your test hermetic, your app should support injection of the networking implementation. During testing, the test injects a fake networking implementation to your app, and this fake implementation will provide seeded data instead of communicating with backend servers.<br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgaFkebvxze2LsbE9WoB9RFQ5oCFivpl5yesX-VYOC-z4tSZWF7AGMEcaZvKsV14eIhH95Gds2QBPB2kXtjHcQAYfzic0rE0MQ8degadJhkfBlRdqyaBVzO8-SXK5bIbn39epaW/s1600/image03.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgaFkebvxze2LsbE9WoB9RFQ5oCFivpl5yesX-VYOC-z4tSZWF7AGMEcaZvKsV14eIhH95Gds2QBPB2kXtjHcQAYfzic0rE0MQ8degadJhkfBlRdqyaBVzO8-SXK5bIbn39epaW/s1600/image03.png" width="226" /></a></div> <br /> <span style="font-size: large;"><b>Strategy 4: Building Apps into Smaller Libraries </b></span><br /> <br /> If you want to scale your app into many modules and views, and plan to add more features while maintaining stable and fast builds/tests, then you should build your app into small components/libraries. Each library should have its own UI resources and user dependency management. This strategy not only enables mocking dependencies of your libraries for hermetic testing, but also serves as an experimentation platform for various components of your application. <br /> <br /> Once you have small components with dependency injection support, you can build a test app for each component. <br /> <br /> The test apps bring up the actual UI of your libraries, fake data needed, and mock dependencies. Espresso tests will run against these test apps. This enables testing of smaller libraries in isolation. <br /> <br /> For example, let’s consider building smaller libraries for login and settings of your app. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj6PvS7P-obesMW8KhNy1g_B27TREbYIdI3-PIN4kVl1K-byl0bZ4uw1XfmKmLWPb9mWMaa57TPwL3XIckIg3DSJ5yFSOZ30yC1QYB_OGcajlLPwHuxEHO1AmN7F7e_RIIIIwmq/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj6PvS7P-obesMW8KhNy1g_B27TREbYIdI3-PIN4kVl1K-byl0bZ4uw1XfmKmLWPb9mWMaa57TPwL3XIckIg3DSJ5yFSOZ30yC1QYB_OGcajlLPwHuxEHO1AmN7F7e_RIIIIwmq/s1600/image00.png" width="242" /></a></div> <br /> The settings component test now looks like: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjpyqMerklLrLKYMxzZWv1-quYtj6_IY6OGVK9Oizp-jvUXTgZX2dQYCbcydjfFaE9LAIYfI_1-p1uBjuGNjhR1QEMzYBz37nTUIYBItefi6Ex47Vh9VL5L8GkaM4TJby6PsDJw/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjpyqMerklLrLKYMxzZWv1-quYtj6_IY6OGVK9Oizp-jvUXTgZX2dQYCbcydjfFaE9LAIYfI_1-p1uBjuGNjhR1QEMzYBz37nTUIYBItefi6Ex47Vh9VL5L8GkaM4TJby6PsDJw/s1600/image01.png" width="227" /></a></div> <br /> <span style="font-size: large;"><b>Conclusion </b></span><br /> <br /> UI testing can be very challenging for rich apps on Android. Here are some UI testing lessons learned on the Google+ team:<br /> <ol style="line-height: 1.3em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Don’t write E2E tests instead of UI tests. Instead write unit tests and integration tests beside the UI tests. </li> <li>Hermetic tests are the way to go. </li> <li>Use dependency injection while designing your app. </li> <li>Build your application into small libraries/modules, and test each one in isolation. You can then have a few integration tests to verify integration between components is correct . </li> <li>Componentized UI tests have proven to be much faster than E2E and 99%+ stable. Fast and stable tests have proven to drastically improve developer productivity.</li> </ol> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by Mona El Mahdy </i><br /> <br /> <span style="font-size: large;"><b>Overview </b></span><br /> <br /> This post reviews four strategies for Android UI testing with the goal of creating UI tests that are fast, reliable, and easy to debug.<br /> <br /> Before we begin, let’s not forget an important rule: whatever can be unit tested should be unit tested. <a href="http://robolectric.org/">Robolectric</a> and <a href="http://tools.android.com/tech-docs/unit-testing-support">gradle unit tests support</a> are great examples of unit test frameworks for Android. UI tests, on the other hand, are used to verify that your application returns the correct UI output in response to a sequence of user actions on a device. <a href="http://developer.android.com/tools/testing-support-library/index.html#Espresso">Espresso</a> is a great framework for running UI actions and verifications in the same process. For more details on the Espresso and UI Automator tools, please see: <a href="http://developer.android.com/tools/testing-support-library/index.html">test support libraries</a>. <br /> <br /> The Google+ team has performed many iterations of UI testing. Below we discuss the lessons learned during each strategy of UI testing. Stay tuned for more posts with more details and code samples. <br /> <br /> <span style="font-size: large;"><b>Strategy 1: Using an End-To-End Test as a UI Test </b></span><br /> <br /> Let’s start with some definitions. A <b>UI test</b> ensures that your application returns the correct UI output in response to a sequence of user actions on a device. An <b>end-to-end (E2E)</b> test brings up the full system of your app including all backend servers and client app. E2E tests will guarantee that data is sent to the client app and that the entire system functions correctly. <br /> <br /> Usually, in order to make the application UI functional, you need data from backend servers, so UI tests need to simulate the data but not necessarily the backend servers. In many cases UI tests are confused with E2E tests because E2E is very similar to manual test scenarios. However, debugging and stabilizing E2E tests is very difficult due to many variables like network flakiness, authentication against real servers, size of your system, etc. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi0PUfVmtCBE3QE8Hi3L7OWM7i5iYLXbqxEalx1R4Q34C788Nk8C9wvWU33qWh6GA6FNJE7guyYHX6Tq3Tfx37uQTZU7CJ5WgBTjh_IECQGWKE2QtoChE7ZHUH_WoLlYBtX45Y2/s1600/image04.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="370" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi0PUfVmtCBE3QE8Hi3L7OWM7i5iYLXbqxEalx1R4Q34C788Nk8C9wvWU33qWh6GA6FNJE7guyYHX6Tq3Tfx37uQTZU7CJ5WgBTjh_IECQGWKE2QtoChE7ZHUH_WoLlYBtX45Y2/s1600/image04.png" width="400" /></a></div> <br /> When you use UI tests as E2E tests, you face the following problems:<br /> <ul style="line-height: 1.3em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Very large and slow tests. </li> <li>High flakiness rate due to timeouts and memory issues. </li> <li>Hard to debug/investigate failures. </li> <li>Authentication issues (ex: authentication from automated tests is very tricky).</li> </ul> <br /> Let’s see how these problems can be fixed using the following strategies.<br /> <br /> <span style="font-size: large;"><b>Strategy 2: Hermetic UI Testing using Fake Servers</b></span><br /> <br /> In this strategy, you avoid network calls and external dependencies, but you need to provide your application with data that drives the UI. Update your application to communicate to a local server rather than external one, and create a fake local server that provides data to your application. You then need a mechanism to generate the data needed by your application. This can be done using various approaches depending on your system design. One approach is to record server responses and replay them in your fake server. <br /> <br /> Once you have hermetic UI tests talking to a local fake server, you should also have <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">server hermetic tests</a>. This way you split your E2E test into a server side test, a client side test, and an integration test to verify that the server and client are in sync (for more details on integration tests, see the backend testing section of <a href="http://googletesting.blogspot.com/2013/08/how-google-team-tests-mobile-apps.html">blog</a>). <br /> <br /> Now, the client test flow looks like: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjmF32nIE84J1wkwAQvjjvvFg-hujwvIC7a922c68jdX7I56pTEbuDvhALwA7U1_rxpVSDLS30EdQZRVtW2JlN5DCSOxl6tWFzLIoT4qhjOE-44OooYUM6NatMO3HxDY8ky4Njf/s1600/image02.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjmF32nIE84J1wkwAQvjjvvFg-hujwvIC7a922c68jdX7I56pTEbuDvhALwA7U1_rxpVSDLS30EdQZRVtW2JlN5DCSOxl6tWFzLIoT4qhjOE-44OooYUM6NatMO3HxDY8ky4Njf/s1600/image02.png" width="201" /></a></div> <br /> While this approach drastically reduces the test size and flakiness rate, you still need to maintain a separate fake server as well as your test. Debugging is still not easy as you have two moving parts: the test and the local server. While test stability will be largely improved by this approach, the local server will cause some flakes. <br /> <br /> Let’s see how this could this be improved... <br /> <br /> <span style="font-size: large;"><b>Strategy 3: Dependency Injection Design for Apps. </b></span><br /> <br /> To remove the additional dependency of a fake server running on Android, you should use dependency injection in your application for swapping real module implementations with fake ones. One example is <a href="http://square.github.io/dagger/">Dagger</a>, or you can create your own dependency injection mechanism if needed. <br /> <br /> This will improve the testability of your app for both unit testing and UI testing, providing your tests with the ability to mock dependencies. In <a href="http://developer.android.com/tools/testing/testing_android.html">instrumentation testing</a>, the test apk and the app under test are loaded in the same process, so the test code has runtime access to the app code. Not only that, but you can also use classpath override (the fact that test classpath takes priority over app under test) to override a certain class and inject test fakes there. For example, To make your test hermetic, your app should support injection of the networking implementation. During testing, the test injects a fake networking implementation to your app, and this fake implementation will provide seeded data instead of communicating with backend servers.<br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgaFkebvxze2LsbE9WoB9RFQ5oCFivpl5yesX-VYOC-z4tSZWF7AGMEcaZvKsV14eIhH95Gds2QBPB2kXtjHcQAYfzic0rE0MQ8degadJhkfBlRdqyaBVzO8-SXK5bIbn39epaW/s1600/image03.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgaFkebvxze2LsbE9WoB9RFQ5oCFivpl5yesX-VYOC-z4tSZWF7AGMEcaZvKsV14eIhH95Gds2QBPB2kXtjHcQAYfzic0rE0MQ8degadJhkfBlRdqyaBVzO8-SXK5bIbn39epaW/s1600/image03.png" width="226" /></a></div> <br /> <span style="font-size: large;"><b>Strategy 4: Building Apps into Smaller Libraries </b></span><br /> <br /> If you want to scale your app into many modules and views, and plan to add more features while maintaining stable and fast builds/tests, then you should build your app into small components/libraries. Each library should have its own UI resources and user dependency management. This strategy not only enables mocking dependencies of your libraries for hermetic testing, but also serves as an experimentation platform for various components of your application. <br /> <br /> Once you have small components with dependency injection support, you can build a test app for each component. <br /> <br /> The test apps bring up the actual UI of your libraries, fake data needed, and mock dependencies. Espresso tests will run against these test apps. This enables testing of smaller libraries in isolation. <br /> <br /> For example, let’s consider building smaller libraries for login and settings of your app. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj6PvS7P-obesMW8KhNy1g_B27TREbYIdI3-PIN4kVl1K-byl0bZ4uw1XfmKmLWPb9mWMaa57TPwL3XIckIg3DSJ5yFSOZ30yC1QYB_OGcajlLPwHuxEHO1AmN7F7e_RIIIIwmq/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj6PvS7P-obesMW8KhNy1g_B27TREbYIdI3-PIN4kVl1K-byl0bZ4uw1XfmKmLWPb9mWMaa57TPwL3XIckIg3DSJ5yFSOZ30yC1QYB_OGcajlLPwHuxEHO1AmN7F7e_RIIIIwmq/s1600/image00.png" width="242" /></a></div> <br /> The settings component test now looks like: <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjpyqMerklLrLKYMxzZWv1-quYtj6_IY6OGVK9Oizp-jvUXTgZX2dQYCbcydjfFaE9LAIYfI_1-p1uBjuGNjhR1QEMzYBz37nTUIYBItefi6Ex47Vh9VL5L8GkaM4TJby6PsDJw/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="400" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjpyqMerklLrLKYMxzZWv1-quYtj6_IY6OGVK9Oizp-jvUXTgZX2dQYCbcydjfFaE9LAIYfI_1-p1uBjuGNjhR1QEMzYBz37nTUIYBItefi6Ex47Vh9VL5L8GkaM4TJby6PsDJw/s1600/image01.png" width="227" /></a></div> <br /> <span style="font-size: large;"><b>Conclusion </b></span><br /> <br /> UI testing can be very challenging for rich apps on Android. Here are some UI testing lessons learned on the Google+ team:<br /> <ol style="line-height: 1.3em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Don’t write E2E tests instead of UI tests. Instead write unit tests and integration tests beside the UI tests. </li> <li>Hermetic tests are the way to go. </li> <li>Use dependency injection while designing your app. </li> <li>Build your application into small libraries/modules, and test each one in isolation. You can then have a few integration tests to verify integration between components is correct . </li> <li>Componentized UI tests have proven to be much faster than E2E and 99%+ stable. Fast and stable tests have proven to drastically improve developer productivity.</li> </ol> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Android UI Automated Testing&url=https://testing.googleblog.com/2015/03/android-ui-automated-testing.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/03/android-ui-automated-testing.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/03/android-ui-automated-testing.html#comments' style='font-weight: 500; text-decoration: underline;'>8 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/03/android-ui-automated-testing.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Mobile' rel='tag'> Mobile </a> , <a class='label' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy' rel='tag'> Mona El Mahdy </a> </span> </div> </div> </div> <div class='post' data-id='4228662649059663258' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/02/the-first-annual-testing-on-toilet.html' itemprop='url' title='The First Annual Testing on the Toilet Awards'> The First Annual Testing on the Toilet Awards </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, February 03, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>By Andrew Trenk </i><br /> <br /> The Testing on the Toilet (TotT) series was <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">created</a> in 2006 as a way to spread unit-testing knowledge across Google by posting flyers in bathroom stalls. It quickly became a part of Google culture and is still going strong today, with new episodes published every week and read in hundreds of bathrooms by thousands of engineers in Google offices across the world. Initially focused on content related to testing, TotT now covers a variety of technical topics, such as tips on writing cleaner code and ways to prevent security bugs. <br /> <br /> While TotT episodes often have a big impact on many engineers across Google, until now we never did anything to formally thank authors for their contributions. To fix that, we decided to honor the most popular TotT episodes of 2014 by establishing the Testing on the Toilet Awards. The winners were chosen through a vote that was open to all Google engineers. The Google Testing Blog is proud to present the winners that were posted on this blog (there were two additional winners that weren’t posted on this blog since we only post testing-related TotT episodes). <br /> <br /> And the winners are ... <br /> <br /> <b>Erik Kuefler:</b> <a href="http://googletesting.blogspot.com/2014/04/testing-on-toilet-test-behaviors-not.html">Test Behaviors, Not Methods</a> and <a href="http://googletesting.blogspot.com/2014/07/testing-on-toilet-dont-put-logic-in.html">Don't Put Logic in Tests</a>  <br /> <b>Alex Eagle:</b> <a href="http://googletesting.blogspot.com/2015/01/testing-on-toilet-change-detector-tests.html">Change-Detector Tests Considered Harmful</a><br /> <br /> The authors of these episodes received their very own Flushy trophy, which they can proudly display on their desks. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhFmo1C9Z3SEL-t4WO2BJvABEJ3slgXEL55Snj8p0wqAssK0jVhrZS1150ZK288RiL4sicmsKmdinDxooNh-DIjQ6qTp1nTp3Zn_gkSz3mKt0-tmkBOoqgOrqZeBd6W6zVdrcg-/s1600/image00.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="320" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhFmo1C9Z3SEL-t4WO2BJvABEJ3slgXEL55Snj8p0wqAssK0jVhrZS1150ZK288RiL4sicmsKmdinDxooNh-DIjQ6qTp1nTp3Zn_gkSz3mKt0-tmkBOoqgOrqZeBd6W6zVdrcg-/s1600/image00.jpg" width="212" /></a></div> <br /> <br /> (The logo on the trophy is the same one we put on the printed version of each TotT episode, which you can see by looking for the “printer-friendly version” link in the <a href="http://googletesting.blogspot.com/search/label/TotT">TotT blog posts</a>). <br /> <br /> Congratulations to the winners! <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>By Andrew Trenk </i><br /> <br /> The Testing on the Toilet (TotT) series was <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">created</a> in 2006 as a way to spread unit-testing knowledge across Google by posting flyers in bathroom stalls. It quickly became a part of Google culture and is still going strong today, with new episodes published every week and read in hundreds of bathrooms by thousands of engineers in Google offices across the world. Initially focused on content related to testing, TotT now covers a variety of technical topics, such as tips on writing cleaner code and ways to prevent security bugs. <br /> <br /> While TotT episodes often have a big impact on many engineers across Google, until now we never did anything to formally thank authors for their contributions. To fix that, we decided to honor the most popular TotT episodes of 2014 by establishing the Testing on the Toilet Awards. The winners were chosen through a vote that was open to all Google engineers. The Google Testing Blog is proud to present the winners that were posted on this blog (there were two additional winners that weren’t posted on this blog since we only post testing-related TotT episodes). <br /> <br /> And the winners are ... <br /> <br /> <b>Erik Kuefler:</b> <a href="http://googletesting.blogspot.com/2014/04/testing-on-toilet-test-behaviors-not.html">Test Behaviors, Not Methods</a> and <a href="http://googletesting.blogspot.com/2014/07/testing-on-toilet-dont-put-logic-in.html">Don't Put Logic in Tests</a>  <br /> <b>Alex Eagle:</b> <a href="http://googletesting.blogspot.com/2015/01/testing-on-toilet-change-detector-tests.html">Change-Detector Tests Considered Harmful</a><br /> <br /> The authors of these episodes received their very own Flushy trophy, which they can proudly display on their desks. <br /> <br /> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhFmo1C9Z3SEL-t4WO2BJvABEJ3slgXEL55Snj8p0wqAssK0jVhrZS1150ZK288RiL4sicmsKmdinDxooNh-DIjQ6qTp1nTp3Zn_gkSz3mKt0-tmkBOoqgOrqZeBd6W6zVdrcg-/s1600/image00.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="320" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhFmo1C9Z3SEL-t4WO2BJvABEJ3slgXEL55Snj8p0wqAssK0jVhrZS1150ZK288RiL4sicmsKmdinDxooNh-DIjQ6qTp1nTp3Zn_gkSz3mKt0-tmkBOoqgOrqZeBd6W6zVdrcg-/s1600/image00.jpg" width="212" /></a></div> <br /> <br /> (The logo on the trophy is the same one we put on the printed version of each TotT episode, which you can see by looking for the “printer-friendly version” link in the <a href="http://googletesting.blogspot.com/search/label/TotT">TotT blog posts</a>). <br /> <br /> Congratulations to the winners! <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:The First Annual Testing on the Toilet Awards&url=https://testing.googleblog.com/2015/02/the-first-annual-testing-on-toilet.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/02/the-first-annual-testing-on-toilet.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/02/the-first-annual-testing-on-toilet.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/02/the-first-annual-testing-on-toilet.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='4545275800686762813' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html' itemprop='url' title='Testing on the Toilet: Change-Detector Tests Considered Harmful'> Testing on the Toilet: Change-Detector Tests Considered Harmful </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Tuesday, January 27, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by Alex Eagle <br /><br /> This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/13k8AsgYdF-2TJx9QIHHvPiuV3JMbsrMLkWmmjdYBLVg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. </i> <br /> <br /> You have just finished refactoring some code without modifying its behavior. Then you run the tests before committing and… a bunch of unit tests are failing. <span style="color: purple;"><b>While fixing the tests, you get a sense that you are wasting time by mechanically applying the same transformation to many tests.</b></span> Maybe you introduced a parameter in a method, and now must update 100 callers of that method in tests to pass an empty string. <br /> <br /> <span style="color: purple;"><b>What does it look like to write tests mechanically?</b></span> Here is an absurd but obvious way:<br /> <pre style="background: rgb(207, 226, 243); border-color: rgb(123, 123, 123); border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #4f8f00;">// Production code:</span> def abs(i: Int) <span style="color: #0433ff;">return</span> (i < 0) ? i * -1 : i <span style="color: #4f8f00;">// Test code:</span> <span style="color: #0433ff;">for</span> (line: String in File(prod_source).read_lines()) <span style="color: #0433ff;">switch</span> (line.number) 1: assert line.content equals "def abs(i: Int)" 2: assert line.content equals " <span style="color: #0433ff;">return</span> (i < 0) ? i * -1 : i"</pre> <br /> <b><span style="color: purple;">That test is clearly not useful</span></b>: it contains an exact copy of the code under test and acts like a checksum. <b><span style="color: purple;">A correct or incorrect program is equally likely to pass</span></b> a test that is a derivative of the code under test. No one is really writing tests like that, but how different is it from this next example?<br /> <pre style="background: rgb(207, 226, 243); border-color: rgb(123, 123, 123); border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #4f8f00;">// Production code:</span> def process(w: Work) firstPart.process(w) secondPart.process(w) <span style="color: #4f8f00;">// Test code:</span> part1 = mock(FirstPart) part2 = mock(SecondPart) w = Work() Processor(part1, part2).process(w) verify_in_order was_called part1.process(w) was_called part2.process(w)</pre> <br /> It is tempting to write a test like this because it requires little thought and will run quickly. <b><span style="color: purple;">This is a change-detector test</span></b>—it is a transformation of the same information in the code under test—and <b><span style="color: purple;">it breaks in response to any change to the production code, without verifying correct behavior</span></b> of either the original or modified production code. <br /> <br /> <b><span style="color: purple;">Change detectors provide negative value</span></b>, since the tests do not catch any defects, and the added maintenance cost slows down development. These tests should be re-written or deleted. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by Alex Eagle <br /><br /> This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/13k8AsgYdF-2TJx9QIHHvPiuV3JMbsrMLkWmmjdYBLVg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. </i> <br /> <br /> You have just finished refactoring some code without modifying its behavior. Then you run the tests before committing and… a bunch of unit tests are failing. <span style="color: purple;"><b>While fixing the tests, you get a sense that you are wasting time by mechanically applying the same transformation to many tests.</b></span> Maybe you introduced a parameter in a method, and now must update 100 callers of that method in tests to pass an empty string. <br /> <br /> <span style="color: purple;"><b>What does it look like to write tests mechanically?</b></span> Here is an absurd but obvious way:<br /> <pre style="background: rgb(207, 226, 243); border-color: rgb(123, 123, 123); border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #4f8f00;">// Production code:</span> def abs(i: Int) <span style="color: #0433ff;">return</span> (i < 0) ? i * -1 : i <span style="color: #4f8f00;">// Test code:</span> <span style="color: #0433ff;">for</span> (line: String in File(prod_source).read_lines()) <span style="color: #0433ff;">switch</span> (line.number) 1: assert line.content equals "def abs(i: Int)" 2: assert line.content equals " <span style="color: #0433ff;">return</span> (i < 0) ? i * -1 : i"</pre> <br /> <b><span style="color: purple;">That test is clearly not useful</span></b>: it contains an exact copy of the code under test and acts like a checksum. <b><span style="color: purple;">A correct or incorrect program is equally likely to pass</span></b> a test that is a derivative of the code under test. No one is really writing tests like that, but how different is it from this next example?<br /> <pre style="background: rgb(207, 226, 243); border-color: rgb(123, 123, 123); border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #4f8f00;">// Production code:</span> def process(w: Work) firstPart.process(w) secondPart.process(w) <span style="color: #4f8f00;">// Test code:</span> part1 = mock(FirstPart) part2 = mock(SecondPart) w = Work() Processor(part1, part2).process(w) verify_in_order was_called part1.process(w) was_called part2.process(w)</pre> <br /> It is tempting to write a test like this because it requires little thought and will run quickly. <b><span style="color: purple;">This is a change-detector test</span></b>—it is a transformation of the same information in the code under test—and <b><span style="color: purple;">it breaks in response to any change to the production code, without verifying correct behavior</span></b> of either the original or modified production code. <br /> <br /> <b><span style="color: purple;">Change detectors provide negative value</span></b>, since the tests do not catch any defects, and the added maintenance cost slows down development. These tests should be re-written or deleted. <br /> <br /> <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Change-Detector Tests Considered Harmful&url=https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Alex%20Eagle' rel='tag'> Alex Eagle </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='post' data-id='2069055255582286813' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2015/01/testing-on-toilet-prefer-testing-public.html' itemprop='url' title='Testing on the Toilet: Prefer Testing Public APIs Over Implementation-Detail Classes'> Testing on the Toilet: Prefer Testing Public APIs Over Implementation-Detail Classes </a> </h2> <div class='post-header'> <div class='published'> <span class='publishdate' itemprop='datePublished'> Wednesday, January 14, 2015 </span> </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <i>by Andrew Trenk <br /><br /> This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1ILEpaxZ9ntMEXNQuL-q4jWHF0QOAMo45elmomn29cZg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. </i> <br /> <br /> <span style="color: purple;"><b>Does this class need to have tests? </b></span><br /> <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #945200;">class</span> UserInfoValidator { <span style="color: #945200;">public</span> <span style="color: #945200;">void</span> validate(UserInfo info) { <span style="color: #945200;">if</span> (info.getDateOfBirth().isInFuture()) { <span style="color: #945200;">throw</span> <span style="color: #945200;">new</span> ValidationException()); } } }</pre> Its method has some logic, so it may be good idea to test it. But <span style="color: purple;"><b>what if its only user looks like this? </b> </span><br /> <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #945200;">public</span> <span style="color: #945200;">class</span> UserInfoService { <span style="color: #945200;">private</span> UserInfoValidator validator; <span style="color: #945200;">public</span> <span style="color: #945200;">void</span> save(UserInfo info) { validator.validate(info); <span style="color: #0096ff;">// Throw an exception if the value is invalid.</span> writeToDatabase(info); } }</pre> The answer is: <span style="color: purple;"><b>it probably doesn’t need tests</b></span>, since all paths can be tested through <span style="font-family: Courier New, Courier, monospace;">UserInfoService</span>. The key distinction is that <span style="color: purple;"><b>the class is an implementation detail, not a public API</b></span>.<br /> <br /> <b><span style="color: purple;">A public API can be called by any number of users</span></b>, who can pass in any possible combination of inputs to its methods. <b><span style="color: purple;">You want to make sure these are well-tested</span></b>, which ensures users won’t see issues when they use the API. Examples of public APIs include classes that are used in a different part of a codebase (e.g., a server-side class that’s used by the client-side) and common utility classes that are used throughout a codebase. <br /> <br /> <b><span style="color: purple;">An implementation-detail class exists only to support public APIs</span></b> and is called by a very limited number of users (often only one). <b><span style="color: purple;">These classes can sometimes be tested indirectly</span></b> by testing the public APIs that use them. <br /> <br /> <b><span style="color: purple;">Testing implementation-detail classes is still useful in many cases</span></b>, such as if the class is complex or if the tests would be difficult to write for the public API class. When you do test them, <b><span style="color: purple;">they often don’t need to be tested in as much depth as a public API,</span></b> since some inputs may never be passed into their methods (in the above code sample, if <span style="font-family: Courier New, Courier, monospace;">UserInfoService</span> ensured that <span style="font-family: Courier New, Courier, monospace;">UserInfo</span> were never <span style="font-family: Courier New, Courier, monospace;">null</span>, then it wouldn’t be useful to test what happens when <span style="font-family: Courier New, Courier, monospace;">null</span> is passed as an argument to <span style="font-family: Courier New, Courier, monospace;">UserInfoValidator.validate</span>, since it would never happen). <br /> <br /> Implementation-detail classes can sometimes be thought of as private methods that happen to be in a separate class, since you typically don’t want to test private methods directly either. <b><span style="color: purple;">You should also try to restrict the visibility of implementation-detail classes,</span></b> such as by making them package-private in Java. <br /> <br /> <b><span style="color: purple;">Testing implementation-detail classes too often leads to a couple problems</span></b>: <br /> <br /> - <b><span style="color: purple;">Code is harder to maintain since you need to update tests more often</span></b>, such as when changing a method signature of an implementation-detail class or even when doing a refactoring. If testing is done only through public APIs, these changes wouldn’t affect the tests at all. <br /> <br /> - <b><span style="color: purple;">If you test a behavior only through an implementation-detail class, you may get false confidence in your code</span></b>, since the same code path may not work properly when exercised through the public API. <b><span style="color: purple;">You also have to be more careful when refactoring</span></b>, since it can be harder to ensure that all the behavior of the public API will be preserved if not all paths are tested through the public API. <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </script> <noscript> <i>by Andrew Trenk <br /><br /> This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1ILEpaxZ9ntMEXNQuL-q4jWHF0QOAMo45elmomn29cZg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. </i> <br /> <br /> <span style="color: purple;"><b>Does this class need to have tests? </b></span><br /> <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #945200;">class</span> UserInfoValidator { <span style="color: #945200;">public</span> <span style="color: #945200;">void</span> validate(UserInfo info) { <span style="color: #945200;">if</span> (info.getDateOfBirth().isInFuture()) { <span style="color: #945200;">throw</span> <span style="color: #945200;">new</span> ValidationException()); } } }</pre> Its method has some logic, so it may be good idea to test it. But <span style="color: purple;"><b>what if its only user looks like this? </b> </span><br /> <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><span style="color: #945200;">public</span> <span style="color: #945200;">class</span> UserInfoService { <span style="color: #945200;">private</span> UserInfoValidator validator; <span style="color: #945200;">public</span> <span style="color: #945200;">void</span> save(UserInfo info) { validator.validate(info); <span style="color: #0096ff;">// Throw an exception if the value is invalid.</span> writeToDatabase(info); } }</pre> The answer is: <span style="color: purple;"><b>it probably doesn’t need tests</b></span>, since all paths can be tested through <span style="font-family: Courier New, Courier, monospace;">UserInfoService</span>. The key distinction is that <span style="color: purple;"><b>the class is an implementation detail, not a public API</b></span>.<br /> <br /> <b><span style="color: purple;">A public API can be called by any number of users</span></b>, who can pass in any possible combination of inputs to its methods. <b><span style="color: purple;">You want to make sure these are well-tested</span></b>, which ensures users won’t see issues when they use the API. Examples of public APIs include classes that are used in a different part of a codebase (e.g., a server-side class that’s used by the client-side) and common utility classes that are used throughout a codebase. <br /> <br /> <b><span style="color: purple;">An implementation-detail class exists only to support public APIs</span></b> and is called by a very limited number of users (often only one). <b><span style="color: purple;">These classes can sometimes be tested indirectly</span></b> by testing the public APIs that use them. <br /> <br /> <b><span style="color: purple;">Testing implementation-detail classes is still useful in many cases</span></b>, such as if the class is complex or if the tests would be difficult to write for the public API class. When you do test them, <b><span style="color: purple;">they often don’t need to be tested in as much depth as a public API,</span></b> since some inputs may never be passed into their methods (in the above code sample, if <span style="font-family: Courier New, Courier, monospace;">UserInfoService</span> ensured that <span style="font-family: Courier New, Courier, monospace;">UserInfo</span> were never <span style="font-family: Courier New, Courier, monospace;">null</span>, then it wouldn’t be useful to test what happens when <span style="font-family: Courier New, Courier, monospace;">null</span> is passed as an argument to <span style="font-family: Courier New, Courier, monospace;">UserInfoValidator.validate</span>, since it would never happen). <br /> <br /> Implementation-detail classes can sometimes be thought of as private methods that happen to be in a separate class, since you typically don’t want to test private methods directly either. <b><span style="color: purple;">You should also try to restrict the visibility of implementation-detail classes,</span></b> such as by making them package-private in Java. <br /> <br /> <b><span style="color: purple;">Testing implementation-detail classes too often leads to a couple problems</span></b>: <br /> <br /> - <b><span style="color: purple;">Code is harder to maintain since you need to update tests more often</span></b>, such as when changing a method signature of an implementation-detail class or even when doing a refactoring. If testing is done only through public APIs, these changes wouldn’t affect the tests at all. <br /> <br /> - <b><span style="color: purple;">If you test a behavior only through an implementation-detail class, you may get false confidence in your code</span></b>, since the same code path may not work properly when exercised through the public API. <b><span style="color: purple;">You also have to be more careful when refactoring</span></b>, since it can be harder to ensure that all the behavior of the public API will be preserved if not all paths are tested through the public API. <span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </span> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Prefer Testing Public APIs Over Implementation-Detail Classes&url=https://testing.googleblog.com/2015/01/testing-on-toilet-prefer-testing-public.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> </span> <span class='fb-custom social-wrapper' data-href='https://www.facebook.com/sharer.php?u=https://testing.googleblog.com/2015/01/testing-on-toilet-prefer-testing-public.html'> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </span> </div> <div class='comment-container'> <i class='comment-img material-icons'>  </i> <a href='https://testing.googleblog.com/2015/01/testing-on-toilet-prefer-testing-public.html#comments' style='font-weight: 500; text-decoration: underline;'>6 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2015/01/testing-on-toilet-prefer-testing-public.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> <span class='labels-caption'> Labels: </span> <span class='labels'> <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </span> </div> </div> </div> <div class='blog-pager' id='blog-pager'> <a class='home-link' href='https://testing.googleblog.com/'> <i class='material-icons'>  </i> </a> <span id='blog-pager-newer-link'> <a class='blog-pager-newer-link' href='https://testing.googleblog.com/search?updated-max=2016-05-27T17:31:00-07:00&max-results=5&reverse-paginate=true' id='Blog1_blog-pager-newer-link' title='Newer Posts'> <i class='material-icons'>  </i> </a> </span> <span id='blog-pager-older-link'> <a class='blog-pager-older-link' href='https://testing.googleblog.com/search?updated-max=2015-01-14T09:30:00-08:00&max-results=5' id='Blog1_blog-pager-older-link' title='Older Posts'> <i class='material-icons'>  </i> </a> </span> </div> <div class='clear'></div> </div></div> </div> </div> <div class='col-right'> <div class='section' id='sidebar-top'><div class='widget HTML' data-version='1' id='HTML8'> <div class='widget-content'> <div class='searchBox'> <input type='text' title='Search This Blog' placeholder='Search blog ...' /> </div> </div> <div class='clear'></div> </div> </div> <div id='aside'> <div class='section' id='sidebar'><div class='widget Label' data-version='1' id='Label1'> <div class='tab'> <img class='sidebar-icon' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII='/> <h2> Labels </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content list-label-widget-content'> <ul> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/TotT'> TotT </a> <span dir='ltr'> 104 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/GTAC'> GTAC </a> <span dir='ltr'> 61 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/James%20Whittaker'> James Whittaker </a> <span dir='ltr'> 42 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Misko%20Hevery'> Misko Hevery </a> <span dir='ltr'> 32 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Code%20Health'> Code Health </a> <span dir='ltr'> 31 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anthony%20Vallone'> Anthony Vallone </a> <span dir='ltr'> 27 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrick%20Copeland'> Patrick Copeland </a> <span dir='ltr'> 23 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jobs'> Jobs </a> <span dir='ltr'> 18 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Andrew%20Trenk'> Andrew Trenk </a> <span dir='ltr'> 13 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/C%2B%2B'> C++ </a> <span dir='ltr'> 11 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund'> Patrik Höglund </a> <span dir='ltr'> 8 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/JavaScript'> JavaScript </a> <span dir='ltr'> 7 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Allen%20Hutchison'> Allen Hutchison </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/George%20Pirocanac'> George Pirocanac </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zhanyong%20Wan'> Zhanyong Wan </a> <span dir='ltr'> 6 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Harry%20Robinson'> Harry Robinson </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Java'> Java </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julian%20Harty'> Julian Harty </a> <span dir='ltr'> 5 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Bender'> Adam Bender </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alberto%20Savoia'> Alberto Savoia </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ben%20Yu'> Ben Yu </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Erik%20Kuefler'> Erik Kuefler </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Philip%20Zembrod'> Philip Zembrod </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shyam%20Seshadri'> Shyam Seshadri </a> <span dir='ltr'> 4 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome'> Chrome </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dillon%20Bly'> Dillon Bly </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Thomas'> John Thomas </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Lesley%20Katzen'> Lesley Katzen </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Kaplan'> Marc Kaplan </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Markus%20Clermont'> Markus Clermont </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander'> Max Kanat-Alexander </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sonal%20Shah'> Sonal Shah </a> <span dir='ltr'> 3 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/APIs'> APIs </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Abhishek%20Arya'> Abhishek Arya </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Myrvold'> Alan Myrvold </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alek%20Icev'> Alek Icev </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Android'> Android </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/April%20Fools'> April Fools </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chaitali%20Narla'> Chaitali Narla </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chris%20Lewis'> Chris Lewis </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome%20OS'> Chrome OS </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Salas'> Diego Salas </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dori%20Reuveni'> Dori Reuveni </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Arbon'> Jason Arbon </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jochen%20Wuttke'> Jochen Wuttke </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kostya%20Serebryany'> Kostya Serebryany </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Eaddy'> Marc Eaddy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87'> Marko Ivanković </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mobile'> Mobile </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Oliver%20Chang'> Oliver Chang </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simon%20Stewart'> Simon Stewart </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stefan%20Kennedy'> Stefan Kennedy </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Flakiness'> Test Flakiness </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Titus%20Winters'> Titus Winters </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tony%20Voellm'> Tony Voellm </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/WebRTC'> WebRTC </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yiming%20Sun'> Yiming Sun </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yvette%20Nameth'> Yvette Nameth </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zuri%20Kemp'> Zuri Kemp </a> <span dir='ltr'> 2 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Aaron%20Jacobs'> Aaron Jacobs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Porter'> Adam Porter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Raider'> Adam Raider </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adel%20Saoud'> Adel Saoud </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Faulkner'> Alan Faulkner </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alex%20Eagle'> Alex Eagle </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Amy%20Fu'> Amy Fu </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anantha%20Keesara'> Anantha Keesara </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Antoine%20Picard'> Antoine Picard </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/App%20Engine'> App Engine </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ari%20Shamash'> Ari Shamash </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Arif%20Sukoco'> Arif Sukoco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Benjamin%20Pick'> Benjamin Pick </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bob%20Nystrom'> Bob Nystrom </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bruce%20Leban'> Bruce Leban </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Arguelles'> Carlos Arguelles </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Israel%20Ortiz%20Garc%C3%ADa'> Carlos Israel Ortiz García </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Cathal%20Weakliam'> Cathal Weakliam </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Christopher%20Semturs'> Christopher Semturs </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Clay%20Murphy'> Clay Murphy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dagang%20Wei'> Dagang Wei </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Maksimovich'> Dan Maksimovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Shi'> Dan Shi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Willemsen'> Dan Willemsen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Chen'> Dave Chen </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Gladfelter'> Dave Gladfelter </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Bendory'> David Bendory </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Mandelberg'> David Mandelberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Derek%20Snyder'> Derek Snyder </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Cavalcanti'> Diego Cavalcanti </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov'> Dmitry Vyukov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz'> Eduardo Bravo Ortiz </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ekaterina%20Kamenskaya'> Ekaterina Kamenskaya </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliott%20Karpilovsky'> Elliott Karpilovsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliotte%20Rusty%20Harold'> Elliotte Rusty Harold </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Espresso'> Espresso </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Felipe%20Sodr%C3%A9'> Felipe Sodré </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Francois%20Aube'> Francois Aube </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Gene%20Volovich'> Gene Volovich </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Google%2B'> Google+ </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goran%20Petrovic'> Goran Petrovic </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goranka%20Bjedov'> Goranka Bjedov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hank%20Duan'> Hank Duan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Havard%20Rast%20Blok'> Havard Rast Blok </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hongfei%20Ding'> Hongfei Ding </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Elbaum'> Jason Elbaum </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Huggins'> Jason Huggins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jay%20Han'> Jay Han </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Hoy'> Jeff Hoy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Listfield'> Jeff Listfield </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jessica%20Tomechak'> Jessica Tomechak </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jim%20Reardon'> Jim Reardon </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joe%20Allan%20Muharsky'> Joe Allan Muharsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joel%20Hynoski'> Joel Hynoski </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Micco'> John Micco </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Penix'> John Penix </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway'> Jonathan Rockway </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez'> Jonathan Velasquez </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Josh%20Armour'> Josh Armour </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julie%20Ralph'> Julie Ralph </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kai%20Kent'> Kai Kent </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kanu%20Tewary'> Kanu Tewary </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Karin%20Lundberg'> Karin Lundberg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kaue%20Silveira'> Kaue Silveira </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Bourrillion'> Kevin Bourrillion </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Graney'> Kevin Graney </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kirkland'> Kirkland </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever'> Kurt Alfred Kluever </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Manjusha%20Parvathaneni'> Manjusha Parvathaneni </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis'> Marek Kiszkis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marius%20Latinis'> Marius Latinis </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Ivey'> Mark Ivey </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Manley'> Mark Manley </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Striebeck'> Mark Striebeck </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Matt%20Lowrie'> Matt Lowrie </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Meredith%20Whittaker'> Meredith Whittaker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Bachman'> Michael Bachman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Klepikov'> Michael Klepikov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Aizatsky'> Mike Aizatsky </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Wacker'> Mike Wacker </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy'> Mona El Mahdy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Noel%20Yap'> Noel Yap </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Palak%20Bansal'> Palak Bansal </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patricia%20Legaspi'> Patricia Legaspi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Per%20Jacobsson'> Per Jacobsson </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht'> Peter Arrenbrecht </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Spragins'> Peter Spragins </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Norman'> Phil Norman </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Rollet'> Phil Rollet </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Pooja%20Gupta'> Pooja Gupta </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Project%20Showcase'> Project Showcase </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Radoslav%20Vasilev'> Radoslav Vasilev </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Dewan'> Rajat Dewan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Jain'> Rajat Jain </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rich%20Martin'> Rich Martin </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Richard%20Bustamante'> Richard Bustamante </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Roshan%20Sembacuttiaratchy'> Roshan Sembacuttiaratchy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov'> Ruslan Khamitov </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sam%20Lee'> Sam Lee </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sean%20Jordan'> Sean Jordan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sebastian%20D%C3%B6rner'> Sebastian Dörner </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sharon%20Zhou'> Sharon Zhou </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shiva%20Garg'> Shiva Garg </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Siddartha%20Janga'> Siddartha Janga </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simran%20Basi'> Simran Basi </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stan%20Chan'> Stan Chan </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stephen%20Ng'> Stephen Ng </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tejas%20Shah'> Tejas Shah </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Analytics'> Test Analytics </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Engineer'> Test Engineer </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tim%20Lyakhovetskiy'> Tim Lyakhovetskiy </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tom%20O%27Neill'> Tom O'Neill </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Vojta%20J%C3%ADna'> Vojta Jína </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/automation'> automation </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/dead%20code'> dead code </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/iOS'> iOS </a> <span dir='ltr'> 1 </span> </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/mutation%20testing'> mutation testing </a> <span dir='ltr'> 1 </span> </li> </ul> <div class='clear'></div> </div> </div><div class='widget BlogArchive' data-version='1' id='BlogArchive1'> <div class='tab'> <i class='material-icons icon'>  </i> <h2> Archive </h2> <i class='material-icons arrow'>  </i> </div> <div class='widget-content'> <div id='ArchiveList'> <div id='BlogArchive1_ArchiveList'> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2025/'> 2025 </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2025/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/'> 2024 </a> <span class='post-count' dir='ltr'>(13)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/'> 2023 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/09/'> Sep </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/'> 2022 </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/'> 2021 </a> <span class='post-count' dir='ltr'>(3)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/'> 2020 </a> <span class='post-count' dir='ltr'>(8)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/'> 2019 </a> <span class='post-count' dir='ltr'>(4)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/'> 2018 </a> <span class='post-count' dir='ltr'>(7)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/05/'> May </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/'> 2017 </a> <span class='post-count' dir='ltr'>(17)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/'> 2016 </a> <span class='post-count' dir='ltr'>(15)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/04/'> Apr </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/'> 2015 </a> <span class='post-count' dir='ltr'>(14)</span> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> <span class='zippy toggle-open'> ▼  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/12/gtac-2015-wrap-up.html'> GTAC 2015 Wrap Up </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/11/gtac-2015-is-next-week.html'> GTAC 2015 is Next Week! </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/10/announcing-gtac-2015-agenda.html'> Announcing the GTAC 2015 Agenda </a> </li> <li> <a href='https://testing.googleblog.com/2015/10/audio-testing-automatic-gain-control.html'> Audio Testing - Automatic Gain Control </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/08/'> Aug </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/08/the-deadline-to-apply-for-gtac-2015-is.html'> The Deadline to Apply for GTAC 2015 is Monday Aug 10 </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/06/'> Jun </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/06/gtac-2015-call-for-proposals-attendance.html'> GTAC 2015: Call for Proposals & Attendance </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/05/gtac-2015-coming-to-cambridge-greater.html'> GTAC 2015 Coming to Cambridge (Greater Boston) in ... </a> </li> <li> <a href='https://testing.googleblog.com/2015/05/multi-repository-development.html'> Multi-Repository Development </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html'> Just Say No to More End-to-End Tests </a> </li> <li> <a href='https://testing.googleblog.com/2015/04/quantum-quality.html'> Quantum Quality </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/03/'> Mar </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/03/android-ui-automated-testing.html'> Android UI Automated Testing </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/02/the-first-annual-testing-on-toilet.html'> The First Annual Testing on the Toilet Awards </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> <ul class='posts'> <li> <a href='https://testing.googleblog.com/2015/01/testing-on-toilet-change-detector-tests.html'> Testing on the Toilet: Change-Detector Tests Consi... </a> </li> <li> <a href='https://testing.googleblog.com/2015/01/testing-on-toilet-prefer-testing-public.html'> Testing on the Toilet: Prefer Testing Public APIs ... </a> </li> </ul> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/'> 2014 </a> <span class='post-count' dir='ltr'>(24)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/12/'> Dec </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/10/'> Oct </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/06/'> Jun </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/02/'> Feb </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/'> 2013 </a> <span class='post-count' dir='ltr'>(16)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/11/'> Nov </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/10/'> Oct </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/08/'> Aug </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/07/'> Jul </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/03/'> Mar </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/'> 2012 </a> <span class='post-count' dir='ltr'>(11)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/12/'> Dec </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/09/'> Sep </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/'> 2011 </a> <span class='post-count' dir='ltr'>(39)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/10/'> Oct </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/09/'> Sep </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/06/'> Jun </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/05/'> May </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/01/'> Jan </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/'> 2010 </a> <span class='post-count' dir='ltr'>(37)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/11/'> Nov </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/10/'> Oct </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/07/'> Jul </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/04/'> Apr </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/03/'> Mar </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/02/'> Feb </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/01/'> Jan </a> <span class='post-count' dir='ltr'>(1)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/'> 2009 </a> <span class='post-count' dir='ltr'>(54)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/12/'> Dec </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/11/'> Nov </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/10/'> Oct </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/08/'> Aug </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/07/'> Jul </a> <span class='post-count' dir='ltr'>(15)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/06/'> Jun </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/05/'> May </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/04/'> Apr </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/'> 2008 </a> <span class='post-count' dir='ltr'>(75)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/12/'> Dec </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/11/'> Nov </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/10/'> Oct </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/09/'> Sep </a> <span class='post-count' dir='ltr'>(8)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/08/'> Aug </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/07/'> Jul </a> <span class='post-count' dir='ltr'>(9)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/06/'> Jun </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/05/'> May </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/04/'> Apr </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/03/'> Mar </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/02/'> Feb </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/01/'> Jan </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/'> 2007 </a> <span class='post-count' dir='ltr'>(41)</span> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/10/'> Oct </a> <span class='post-count' dir='ltr'>(6)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/09/'> Sep </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/08/'> Aug </a> <span class='post-count' dir='ltr'>(3)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/07/'> Jul </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/06/'> Jun </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/05/'> May </a> <span class='post-count' dir='ltr'>(2)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/04/'> Apr </a> <span class='post-count' dir='ltr'>(7)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/03/'> Mar </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/02/'> Feb </a> <span class='post-count' dir='ltr'>(5)</span> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> <span class='zippy'> ►  </span> </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/01/'> Jan </a> <span class='post-count' dir='ltr'>(4)</span> </li> </ul> </li> </ul> </div> </div> <div class='clear'></div> </div> </div><div class='widget HTML' data-version='1' id='HTML6'> <div class='widget-content'> <a href="http://googletesting.blogspot.com/atom.xml"> <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==" class="sidebar-icon" /> <h2>Feed</h2> </a> </div> <div class='clear'></div> </div></div> <div class='no-items section' id='sidebar-bottom'></div> </div> </div> <div style='clear:both;'></div> </div>  <div class='google-footer-outer loading'> <div id='google-footer'> <a href='//www.google.com/'> <img class='google-logo-dark' height='36' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==' style='margin-top: -16px;' width='92'/> </a> <ul> <li> <a href='//www.google.com/'> Google </a> </li> <li> <a href='//www.google.com/policies/privacy/'> Privacy </a> </li> <li> <a href='//www.google.com/policies/terms/'> Terms </a> </li> </ul> </div> </div> <script type='text/javascript'> //<![CDATA[ // Social sharing popups. var postEl = document.getElementsByClassName('social-wrapper'); var postCount = postEl.length; for(i=0; i<postCount;i++){ postEl[i].addEventListener("click", function(event){ var postUrl = this.getAttribute("data-href"); window.open( postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); });} //]]> </script> <script type='text/javascript'> //<![CDATA[ var BreakpointHandler = function() { this.initted = false; this.isHomePage = false; this.isMobile = false; }; BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) { // Use $.trim for IE8 compatibility summaryHtml = $.trim(summaryHtml).replace(/(<br>|\s)+$/,''); if (lastNode.nodeType == 3) { var lastChar = summaryHtml.slice(-1); if (!lastChar.match(/[.”"?]/)) { if (!lastChar.match(/[A-Za-z]/)) { summaryHtml = summaryHtml.slice(0, -1); } summaryHtml += ' ...'; } } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) { summaryHtml += ' ...'; } return summaryHtml; }; BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) { var seenWords = 0; var summaryHtml = ''; for (var i=0; i < content.childNodes.length; i++) { var node = content.childNodes[i]; var nodeText; if (node.nodeType == 1) { if (node.hasAttribute('data-about-pullquote')) { continue; } nodeText = node.textContent; if (nodeText === undefined) { // innerText for IE8 nodeText = node.innerText; } if (node.nodeName == 'DIV' || node.nodeName == 'B') { // Don't end early if we haven't seen enough words. if (seenWords < 10) { continue; } if (i > 0) { summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]); } break; } summaryHtml += node.outerHTML; } else if (node.nodeType == 3) { nodeText = node.nodeValue; summaryHtml += nodeText + ' '; } var words = nodeText.match(/\S+\s*/g); if (!words) { continue; } var remain = numWords - seenWords; if (words.length >= remain) { summaryHtml = this.finalizeSummary(summaryHtml, node); break; } seenWords += words.length; } return summaryHtml; }; BreakpointHandler.prototype.detect = function() { var match, pl = /\+/g, search = /([^&=]+)=?([^&]*)/g, decode = function (s) { return decodeURIComponent(s.replace(pl, " ")); }, query = window.location.search.substring(1); var urlParams = {}; while (match = search.exec(query)) urlParams[decode(match[1])] = decode(match[2]); this.isListPage = $('html').hasClass('list-page'); this.isMobile = urlParams['m'] === '1'; this.isHomePage = window.location.pathname == '/'; }; BreakpointHandler.prototype.initContent = function() { var self = this; $('.post').each(function(index) { var body = $(this).children('.post-body')[0]; var content = $(body).children('.post-content')[0]; $(content).addClass('post-original'); var data = $(content).children('script').html(); data = self.rewriteForSSL(data); // If exists, extract specified editor's preview. var match = data.match(/([\s\S]+?)<div data-is-preview.+?>([\s\S]+)<\/div>/m); if (match) { data = match[1]; } // Prevent big images from loading when they aren't needed. // This must be done as a pre-injection step, since image loading can't be // canceled once embedded into the DOM. if (self.isListPage && self.isMobile) { data = data.replace(/<(img|iframe) .+?>/g, ''); } // Insert template to be rendered as nodes. content.innerHTML = data; if (self.isListPage) { var summary = document.createElement('div'); $(summary).addClass('post-content'); $(summary).addClass('post-summary'); body.insertBefore(summary, content); if (match) { // Use provided summary. summary.innerHTML = match[2]; } else { // Generate a summary. // Summary generation relies on DOM, so it must occur after content is // inserted into the page. summary.innerHTML = self.generateSummaryFromContent(content, 30); } // Add read more link to summary. var titleAnchor = $(this).find('.title a')[0]; var link = titleAnchor.cloneNode(true); link.innerHTML = 'Read More'; $(link).addClass('read-more'); summary.appendChild(link); } }); // Firefox does not allow for proper styling of BR. if (navigator.userAgent.indexOf('Firefox') > -1) { $('.post-content br').replaceWith('<span class="space"></span>'); } $('.loading').removeClass('loading'); }; BreakpointHandler.prototype.process = function() { if (!this.initted) { var makeInsecureImageRegex = function(hosts) { var whitelist = hosts.join('|').replace(/\./g,'\\.'); // Normal image tags, plus input images (yes, this is possible!) return new RegExp('(<(img|input)[^>]+?src=("|\'))http:\/\/(' + whitelist +')', 'g'); }; this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS); this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]); this.detect(); this.initContent(); this.initted = true; } }; BreakpointHandler.KNOWN_HTTPS_HOSTS = [ "www.google.org", "www.google.com", "services.google.com", "blogger.com", "draft.blogger.com", "www.blogger.com", "photos1.blogger.com", "photos2.blogger.com", "photos3.blogger.com", "blogblog.com", "img1.blogblog.com", "img2.blogblog.com", "www.blogblog.com", "www1.blogblog.com", "www2.blogblog.com", "0.bp.blogspot.com", "1.bp.blogspot.com", "2.bp.blogspot.com", "3.bp.blogspot.com", "4.bp.blogspot.com", "lh3.googleusercontent.com", "lh4.googleusercontent.com", "lh5.googleusercontent.com", "lh6.googleusercontent.com", "themes.googleusercontent.com", ]; BreakpointHandler.prototype.rewriteForSSL = function(html) { // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content. return html.replace(this.sslImageRegex, '$1https://$4') .replace(this.sslImageCurrentDomainRegex, '$1//$4') .replace(/(<(embed|iframe)[^>]+?src=("|'))http:\/\/([^"']*?(youtube|picasaweb\.google)\.com)/g, '$1https://$4') // Slideshow SWF takes a image host, so we need to rewrite that parameter. .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s'); }; $(document).ready(function() { var handler = new BreakpointHandler(); handler.process(); // Top-level navigation. $(".BlogArchive .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); $(".Label .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); // Blog archive year expansion. $('.BlogArchive .intervalToggle').click(function(ev) { ev.preventDefault(); if ($(this).parent().hasClass('collapsed')) { $(this).parent().removeClass('collapsed'); $(this).parent().addClass('expanded'); } else { $(this).parent().removeClass('expanded'); $(this).parent().addClass('collapsed'); } }); // Reverse order of months. $('.BlogArchive .intervalToggle + div').each(function(_, items) { var year = $(this); year.children().each(function(_, month) { year.prepend(month); }); }); // Set anchors to open in new tab. $('.post-content img').parent().each(function(_, node) { if (node.nodeName == 'A') { $(this).attr('target', '_blank'); } }); // Process search requests. $('.searchBox input').on("keypress", function(ev) { if (ev.which == 13) { window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val()); } }); }); (function($, window) { var archiveButton = $($('#sidebar .widget.BlogArchive h2')[0]); var folderIcon = $('#sidebar .widget.BlogArchive h2::after'); var archivePanel = $('#BlogArchive1'); archiveButton.click(function(e) { if (archivePanel.hasClass('archive-open')) { // It's open, so now we close it archivePanel.removeClass('archive-open'); archivePanel.addClass('archive-closed'); archiveButton.removeClass('archive-open'); archiveButton.addClass('archive-closed'); archivePanel.css('height', '60px'); } else { // It's closed, so open it archivePanel.removeClass('archive-closed'); archivePanel.addClass('archive-open'); archiveButton.removeClass('archive-closed'); archiveButton.addClass('archive-open'); archivePanel.css('height', 'auto'); folderIcon.css('content', 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEggkDzi8QVSua_XxLOnfxWE_8nQc1MAhRPBxJej91JW2ZBbKvb2TjaNf66ZoTqCrBivwRr-TLJYxFm2ZAiMd-zfbJ0TjblXGvUszWFbdRncNOL7jseiJCL9uQoqN8BM8FpesdSw/s1600/keyboard_arrow_up_grey600_24dp.png'); } }); })($, window); (function ($, window, devsite) { devsite.devsite = {}; devsite.expandLeftNav = devsite.expandLeftNav || {}; devsite.expandLeftNav.ANALYTICS_LABEL_ = 'Hamburger menu'; devsite.expandLeftNav.ANALYTICS_ACTION_OPEN_ = 'Open'; devsite.expandLeftNav.ANALYTICS_ACTION_CLOSE_ = 'Close'; devsite.expandLeftNav.lastState_ = 0; devsite.expandLeftNav.chekovEnabled_ = false; devsite.expandLeftNav.init = function () { devsite.expandLeftNav.chekovEnabled_ = true; var navElement; if (devsite.expandLeftNav.chekovEnabled_) { navElement = $('.devsite-nav-responsive'); } else { navElement = $('.devsite-section-nav-responsive'); } this.drawerWidth = parseInt(navElement.css('width')); navElement.css({ 'left': -(this.drawerWidth) }); var expandButton = $('.devsite-expand-section-nav'); expandButton.click(function () { devsite.expandLeftNav.handleNavigationOpened(); }); navElement.find('.devsite-nav-responsive-forward').click( devsite.expandLeftNav.openPanel); navElement.find('.devsite-nav-responsive-back').click( devsite.expandLeftNav.closePanel); if ($('.devsite-nav-responsive-tabs-panel + ' + '.devsite-nav-responsive-sidebar-panel').length) { devsite.expandLeftNav.openPanel(); } }; devsite.expandLeftNav.handleNavigationOpened = function (opt_noAnimate) { var mask; if (devsite.expandLeftNav.chekovEnabled_) { var nav = $('.devsite-nav-responsive'); if (opt_noAnimate) { nav.addClass('devsite-nav-responsive-no-animate'); mask = devsite.devsite.showSiteMask(0); } else { mask = devsite.devsite.showSiteMask(); } nav.addClass('devsite-nav-responsive-open'); } else { devsite.expandLeftNav.lastState_ = devsite.sticky.currentState; devsite.sticky.setState(devsite.sticky.state.COLLAPSED_HEADER); var headerHeight = devsite.sticky.getHeaderHeight() - devsite.sticky.desiredMargin; var drawerHeight = $(window).height() - headerHeight; mask = devsite.devsite.showArticleMask(); $('.devsite-section-nav-responsive').css({ 'height': drawerHeight, 'left': '0', 'top': headerHeight, 'visibility': 'visible' }).focus(); } $(mask).click(devsite.expandLeftNav.handleNavigationClosed); $(document).on('keydown.escape', function (e) { if (e.keyCode == $.ui.keyCode.ESCAPE) { devsite.expandLeftNav.handleNavigationClosed(); $(document).off('keydown.escape'); } }); }; devsite.expandLeftNav.getScrollTop = function () { return $(window).scrollTop(); }; devsite.expandLeftNav.handleNavigationClosed = function () { var nav; if (devsite.expandLeftNav.chekovEnabled_) { nav = $('.devsite-nav-responsive'); devsite.devsite.hideSiteMask(); nav.removeClass('devsite-nav-responsive-open ' + 'devsite-nav-responsive-no-animate'); } else { nav = $('.devsite-section-nav-responsive'); devsite.devsite.hideArticleMask(); devsite.sticky.setState(devsite.expandLeftNav.lastState_); nav .css('left', -(devsite.expandLeftNav.drawerWidth)) .one('transitionend', function () { nav.css('visibility', 'hidden'); }); } nav.scrollTop(0); }; devsite.expandLeftNav.openPanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel.show(); parentPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel .removeClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); parentPanel.hide(); }); setTimeout(function () { childPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); }, 1); }; devsite.expandLeftNav.closePanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); parentPanel .show() .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); setTimeout(function () { parentPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); childPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel.hide(); parentPanel.removeClass('devsite-nav-responsive-transition'); }); }, 1); }; devsite.expandLeftNav.FADE_SLOW_ = 'slow'; devsite.expandLeftNav.FADE_FAST_ = 'fast'; devsite.expandLeftNav.SITE_MASK_CSS_ = '.devsite-site-mask'; devsite.devsite.showSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_SLOW_; } devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, false, opt_animate); }; devsite.devsite.hideSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_FAST_; } devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, true, opt_animate); }; devsite.devsite.showArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_('.devsite-article-mask', false, devsite.expandLeftNav.FADE_SLOW_); }; devsite.devsite.hideArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_('.devsite-article-mask', true, devsite.expandLeftNav.FADE_FAST_); }; devsite.devsite.setMask_ = function(className, out, opt_fadeTime) { var query = $(className); if (opt_fadeTime === 0) { out ? query.hide() : query.show(); } else { out ? query.fadeOut(opt_fadeTime) : query.fadeIn(opt_fadeTime); } return $(className)[0]; }; devsite.devsite.setMouseScrollingEnabled = function(trueOrFalse) { if (trueOrFalse == true) { $('html, body').css({ 'overflow': '' }); } else { $('html, body').css({ 'overflow': 'hidden' }); } }; })($, window, devsite = {}); if (window.jQuery) { $(document).ready(function () { if (window.devsite) { devsite.expandLeftNav.init(); } }); } //]]> </script> <style> .widget ul{ line-height: 1.6 !important; } #sidebar ul li a{ color:black; line-height: 20px; } #sidebar ul li a:hover{ color:#4184F3; } </style> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/1455187647-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY77VRurdiubBA3Xa6dMNVApOAu47Q:1743600893660';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d15045980','//testing.googleblog.com/2015/','15045980'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '15045980', 'title': 'Google Testing Blog', 'url': 'https://testing.googleblog.com/2015/', 'canonicalUrl': 'https://testing.googleblog.com/2015/', 'homepageUrl': 'https://testing.googleblog.com/', 'searchUrl': 'https://testing.googleblog.com/search', 'canonicalHomepageUrl': 'https://testing.googleblog.com/', 'blogspotFaviconUrl': 'https://testing.googleblog.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'G-838ZCPQWM6', 'analytics4': true, 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Google Testing Blog - RSS\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/15045980/posts/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/88857c833b4750f1', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'archive', 'pageName': '2015', 'pageTitle': 'Google Testing Blog: 2015'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Google Testing Blog', 'description': '', 'url': 'https://testing.googleblog.com/2015/', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': true, 'isLabelSearch': false, 'archive': {'year': 2015, 'rangeMessage': 'Showing posts from 2015'}}}]); _WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/1425802369-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull')); </script> </body> </html>