CINXE.COM

<!DOCTYPE html> <html class='v2 list-page' dir='ltr' itemscope='' itemtype='http://schema.org/Blog' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <title> Google Testing Blog: 2014 </title> <meta content='width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0' name='viewport'/> <meta content='IE=Edge' http-equiv='X-UA-Compatible'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='en_US' property='og:locale'/> <meta content='https://testing.googleblog.com/2014/' property='og:url'/> <meta content='Google Testing Blog' property='og:site_name'/>  <meta content='Google Testing Blog' property='og:title'/> <meta content='summary' name='twitter:card'/> <meta content='@googletesting' name='twitter:creator'/> <link href='https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic' rel='stylesheet' type='text/css'/> <link href='https://fonts.googleapis.com/icon?family=Material+Icons' rel='stylesheet'/> <script src='https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js' type='text/javascript'></script>  <style id='page-skin-1' type='text/css'></style> <style id='template-skin-1' type='text/css'></style>  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://testing.googleblog.com/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://testing.googleblog.com/2014/' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://testing.googleblog.com/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="Google Testing Blog - RSS" href="https://testing.googleblog.com/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://draft.blogger.com/feeds/15045980/posts/default" />  <meta content='https://testing.googleblog.com/2014/' property='og:url'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='' property='og:description'/>  <base target='_self'/> <style> html { font-family: Roboto, sans-serif; -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; } body { padding: 0; /* This ensures that the scroll bar is always present, which is needed */ /* because content render happens after page load; otherwise the header */ /* would "bounce" in-between states. */ min-height: 150%; } h2 { font-size: 20px; } h1, h2, h3, h4, h5 { line-height: 2em; } html, h4, h5, h6 { font-size: 17px; } h3 { font-size: 18px !important; } a, a:visited { color: #4184F3; text-decoration: none; } a:focus, a:hover, a:active { text-decoration: none; } .Header { margin-top: 15px; } .Header h1 { font-size: 32px; font-weight: 300; line-height: 32px; height: 42px; } .header-inner .Header .titlewrapper { padding: 0; margin-top: 30px; } .header-inner .Header .descriptionwrapper { padding: 0; margin: 0; } .cols-wrapper { margin-top: 56px; } .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 60px; } .header-inner { height: 256px; position: relative; } html, .header-inner a { color: #212121; color: rgba(0,0,0,.87); } .header-inner .google-logo { display: inline-block; background-size: contain; z-index: 1; height: 46px; overflow: hidden; margin-top: 4px; margin-right: 8px; } .header-left { position: absolute; top: 50%; -webkit-transform: translateY(-50%); transform: translateY(-50%); margin-top: 12px; width: 100%; } .google-logo { margin-left: -4px; } #google-footer { position: relative; font-size: 13px; list-style: none; text-align: right; } #google-footer a { color: #444; } #google-footer ul { margin: 0; padding: 0; height: 144px; line-height: 144px; } #google-footer ul li { display: inline; } #google-footer ul li:before { color: #999; content: "\00b7"; font-weight: bold; margin: 5px; } #google-footer ul li:first-child:before { content: ''; } #google-footer .google-logo-dark { left: 0; margin-top: -16px; position: absolute; top: 50%; } /** Sitemap links. **/ .footer-inner-2 { font-size: 14px; padding-top: 42px; padding-bottom: 74px; } .footer-inner-2 .HTML h2 { color: #212121; color: rgba(0,0,0,.87); font-size: 14px; font-weight: 500; padding-left: 0; margin: 10px 0; } .footer-inner-2 .HTML ul { font-weight: normal; list-style: none; padding-left: 0; } .footer-inner-2 .HTML li { line-height: 24px; padding: 0; } .footer-inner-2 li a { color: rgba(65,132,243,.87); } /** Archive widget. **/ .BlogArchive { font-size: 13px; font-weight: normal; } .BlogArchive .widget-content { display: none; } .BlogArchive h2, .Label h2 { color: #4184F3; text-decoration: none; } .BlogArchive .hierarchy li { display: inline-block; } /* Specificity needed here to override widget CSS defaults. */ .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li { padding-left: 0; text-indent: 0; } .BlogArchive .intervalToggle { cursor: pointer; } .BlogArchive .expanded .intervalToggle .new-toggle { -ms-transform: rotate(180deg); transform: rotate(180deg); } .BlogArchive .new-toggle { float: right; padding-top: 3px; opacity: 0.87; } #ArchiveList { text-transform: uppercase; } #ArchiveList .expanded > ul:last-child { margin-bottom: 16px; } #ArchiveList .archivedate { width: 100%; } /* Months */ .BlogArchive .items { max-width: 150px; margin-left: -4px; } .BlogArchive .expanded .items { margin-bottom: 10px; overflow: hidden; } .BlogArchive .items > ul { float: left; height: 32px; } .BlogArchive .items a { padding: 0 4px; } .Label { font-size: 13px; font-weight: normal; } .sidebar-icon { display: inline-block; width: 24px; height: 24px; vertical-align: middle; margin-right: 12px; margin-top: -1px } .Label a { margin-right: 4px; } .Label .widget-content { display: none; } .FollowByEmail { font-size: 13px; font-weight: normal; } .FollowByEmail h2 { background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); background-repeat: no-repeat; background-position: 0 50%; text-indent: 30px; } .FollowByEmail .widget-content { display: none; } .searchBox input { border: 1px solid #eee; color: #212121; color: rgba(0,0,0,.87); font-size: 14px; padding: 8px 8px 8px 40px; width: 164px; font-family: Roboto, sans-serif; background: url("https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png") 8px center no-repeat; } .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */ color: rgba(0,0,0,.54); } .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */ color: #000; opacity: 0.54; } .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */ color: #000; opacity: 0.54; } .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */ color: #757575; } .widget-item-control { margin-top: 0px; } .section { margin: 0; padding: 0; } #sidebar-top { border: 1px solid #eee; } #sidebar-top > div { margin: 16px 0; } .widget ul { line-height: 1.6; } /*main post*/ .post { margin-bottom:30px; } #main .post .title { margin: 0; } #main .post .title a { color: #212121; color: rgba(0,0,0,.87); font-weight: normal; font-size: 24px; } #main .post .title a:hover { text-decoration:none; color:#4184F3; } .message, #main .post .post-header { margin: 0; padding: 0; } #main .post .post-header .caption, #main .post .post-header .labels-caption, #main .post .post-footer .caption, #main .post .post-footer .labels-caption { color: #444; font-weight: 500; } #main .tr-caption-container td { text-align: left; } #main .post .tr-caption { color: #757575; color: rgba(0,0,0,.54); display: block; max-width: 560px; padding-bottom: 20px; } #main .post .tr-caption-container { line-height: 24px; margin: -1px 0 0 0 !important; padding: 4px 0; text-align: left; } #main .post .post-header .published{ font-size:11px; font-weight:bold; } .post-header .publishdate { font-size: 17px; font-weight:normal; color: #757575; color: rgba(0,0,0,.54); } #main .post .post-footer{ font-size:12px; padding-bottom: 21px; } .label-footer { margin-bottom: 12px; margin-top: 12px; } .comment-img { margin-right: 16px; opacity: 0.54; vertical-align: middle; } #main .post .post-header .published { margin-bottom: 40px; margin-top: -2px; } .post .post-content { color: #212121; color: rgba(0,0,0,.87); font-size: 17px; margin: 25px 0 36px 0; line-height: 32px; word-wrap: break-word; } .post-body .post-content ul, .post-body .post-content ol { margin: 16px 0; padding: 0 48px; } .post-summary { display: none; } /* Capture section headers. */ .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b { display: inline-block; margin-bottom: 8px; margin-top: 24px; } .post-content li { line-height: 32px; } /* Override all post images/videos to left align. */ .post-content .separator, .post-content > div { text-align: left; } .post-content .separator > a, .post-content .separator > span { margin-left: 0 !important; } .post-content img { max-width: 100%; } .post-content .tr-caption-container img { margin-bottom: 12px; } .post-content iframe, .post-content embed { max-width: 100%; } .post-content .carousel-container { margin-bottom: 48px; } #main .post-content b { font-weight: 500; } /* These are the main paragraph spacing tweaks. */ #main .post-content br { content: ' '; display: block; padding: 4px; } .post-content .space { display: block; height: 8px; } .post-content iframe + .space, .post-content iframe + br { padding: 0 !important; } #main .post .jump-link { margin-bottom:10px; } .post-content img, .post-content iframe { margin: 30px 0 20px 0; } .post-content > img:first-child, .post-content > iframe:first-child { margin-top: 0; } .col-right .section { padding: 0 16px; } #aside { background:#fff; border:1px solid #eee; border-top: 0; } #aside .widget { margin:0; } #aside .widget h2, #ArchiveList .toggle + a.post-count-link { color: #212121; color: rgba(0,0,0,.87); font-weight: 400 !important; margin: 0; } #ArchiveList .toggle { float: right; } #ArchiveList .toggle .material-icons { padding-top: 4px; } #sidebar .tab { cursor: pointer; } #sidebar .tab .arrow { display: inline-block; float: right; } #sidebar .tab .icon { display: inline-block; vertical-align: top; height: 24px; width: 24px; margin-right: 13px; margin-left: -1px; margin-top: 1px; color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget-content > :first-child { padding-top: 8px; } #sidebar .active .tab .arrow { -ms-transform: rotate(180deg); transform: rotate(180deg); } #sidebar .arrow { color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget h2 { font-size: 14px; line-height: 24px; display: inline-block; } #sidebar .widget .BlogArchive { padding-bottom: 8px; } #sidebar .widget { border-bottom: 1px solid #eee; box-shadow: 0px 1px 0 white; margin-bottom: 0; padding: 14px 0; min-height: 20px; } #sidebar .widget:last-child { border-bottom: none; box-shadow: none; margin-bottom: 0; } #sidebar ul { margin: 0; padding: 0; } #sidebar ul li { list-style:none; padding:0; } #sidebar ul li a { line-height: 32px; } #sidebar .archive { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC"); height: 24px; line-height: 24px; padding-left: 30px; } #sidebar .labels { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC"); height: 20px; line-height: 20px; padding-left: 30px; } #sidebar .rss a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII="); } #sidebar .subscription a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); } #sidebar-bottom { background: #f5f5f5; border-top:1px solid #eee; } #sidebar-bottom .widget { border-bottom: 1px solid #e0e0e0; padding: 15px 0; text-align: center; } #sidebar-bottom > div:last-child { border-bottom: 0; } #sidebar-bottom .text { line-height: 20px; } /* Home, forward, and backward pagination. */ .blog-pager { border-top : 1px #e0e0e0 solid; padding-top: 10px; margin-top: 15px; text-align: right !important; } #blog-pager { margin-botom: 0; margin-top: -14px; padding: 16px 0 0 0; } #blog-pager a { display: inline-block; } .blog-pager i.disabled { opacity: 0.2 !important; } .blog-pager i { color: black; margin-left: 16px; opacity: 0.54; } .blog-pager i:hover, .blog-pager i:active { opacity: 0.87; } #blog-pager-older-link, #blog-pager-newer-link { float: none; } .gplus-profile { background-color: #fafafa; border: 1px solid #eee; overflow: hidden; width: 212px; } .gplus-profile-inner { margin-left: -1px; margin-top: -1px; } /* Sidebar follow buttons. */ .followgooglewrapper { padding: 12px 0 0 0; } .loading { visibility: hidden; } .detail-page .post-footer .cmt_iframe_holder { padding-top: 40px !important; } /** Desktop **/ @media (max-width: 900px) { .col-right { display: none; } .col-main { margin-right: 0; min-width: initial; } .footer-outer { display: none; } .cols-wrapper { min-width: initial; } .google-footer-outer { background-color: #f5f5f5; } } /** Tablet **/ @media (max-width: 712px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 40px; } } /* An extra breakpoint accommodating for long blog titles. */ @media (max-width: 600px) { .header-left { height: 100%; position: initial; top: inherit; margin-top: 0; -webkit-transform: initial; transform: initial; } .header-title { margin-top: 18px; } .header-inner { height: auto; margin-bottom: 32px; margin-top: 32px; } .header-desc { margin-top: 12px; } .header-inner .google-logo { height: 40px; margin-top: 3px; } .header-inner .google-logo img { height: 42px; } .header-title h2 { font-size: 32px; line-height: 40px; } } /** Mobile/small desktop window; also landscape. **/ @media (max-width: 480px), (max-height: 480px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 16px; } .cols-wrapper { margin-top: 0; } .post-header .publishdate, .post .post-content { font-size: 16px; } .post .post-content { line-height: 28px; margin-bottom: 30px; } .post { margin-top: 30px; } .byline-author { display: block; font-size: 12px; line-height: 24px; margin-top: 6px; } #main .post .title a { font-weight: 500; color: #4c4c4c; color: rgba(0,0,0,.70); } #main .post .post-header { padding-bottom: 12px; } #main .post .post-header .published { margin-bottom: -8px; margin-top: 3px; } .post .read-more { display: block; margin-top: 14px; } .post .tr-caption { font-size: 12px; } #main .post .title a { font-size: 20px; line-height: 30px; } .post-content iframe { /* iframe won't keep aspect ratio when scaled down. */ max-height: 240px; } .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe { margin-left: -16px; max-width: inherit; width: calc(100% + 32px); } .post-content table, .post-content td { width: 100%; } #blog-pager { margin: 0; padding: 16px 0; } /** List page tweaks. **/ .list-page .post-original { display: none; } .list-page .post-summary { display: block; } .list-page .comment-container { display: none; } .list-page #blog-pager { padding-top: 0; border: 0; margin-top: -8px; } .list-page .label-footer { display: none; } .list-page #main .post .post-footer { border-bottom: 1px solid #eee; margin: -16px 0 0 0; padding: 0 0 20px 0; } .list-page .post .share { display: none; } /** Detail page tweaks. **/ .detail-page .post-footer .cmt_iframe_holder { padding-top: 32px !important; } .detail-page .label-footer { margin-bottom: 0; } .detail-page #main .post .post-footer { padding-bottom: 0; } .detail-page #comments { display: none; } } [data-about-pullquote], [data-is-preview], [data-about-syndication] { display: none; } </style> <noscript> <style> .loading { visibility: visible }</style> </noscript>  <script async='true' src='https://www.googletagmanager.com/gtag/js?id=G-838ZCPQWM6'></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-838ZCPQWM6'); </script> <link href='https://draft.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://draft.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=4804591f-4b0d-457d-91c1-fd6e244b873a' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <script type='text/javascript'> //<![CDATA[ var axel = Math.random() + ""; var a = axel * 10000000000000; document.write('<iframe src="https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?" width="1" height="1" frameborder="0" style="display:none"></iframe>'); //]]> </script> <noscript> <img alt='' height='1' src='https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?' width='1'/> </noscript>  <div class='header-outer'> <div class='header-inner'> <div class='section' id='header'><div class='widget Header' data-version='1' id='Header1'> <div class='header-left'> <div class='header-title'> <a class='google-logo' href='https://testing.googleblog.com/'> <img height='50' src='https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png'/> </a> <a href='/.'> <h2> Testing Blog </h2> </a> </div> <div class='header-desc'> </div> </div> </div></div> </div> </div>  <div class='cols-wrapper loading'> <div class='col-main-wrapper'> <div class='col-main'> <div class='section' id='main'><div class='widget Blog' data-version='1' id='Blog1'> <div class='post' data-id='6651793442349240032' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/12/testing-on-toilet-truth-fluent.html' itemprop='url' title='Testing on the Toilet: Truth: a fluent assertion framework'> Testing on the Toilet: Truth: a fluent assertion framework </a> </h2> <div class='post-header'> <div class='published'> Friday, December 19, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Dori Reuveni and Kurt Alfred Kluever This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1TyLthjSGeELWVloxx2hI9l57pNRSN4UVAaNdB7H4Ctg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. As engineers, we spend most of our time reading existing code, rather than writing new code. Therefore, we must make sure we always write clean, readable code. The same goes for our tests; we need a way to clearly express our test assertions. Truth is an open source, fluent testing framework for Java designed to make your test assertions and failure messages more readable. The fluent API makes reading (and writing) test assertions much more natural, prose-like, and discoverable in your IDE via autocomplete. For example, compare how the following assertion reads with JUnit vs. Truth: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">assertEquals("March", monthMap.get(3)); // JUnit assertThat(monthMap).containsEntry(3, "March"); // Truth</pre> Both statements are asserting the same thing, but the assertion written with Truth can be easily read from left to right, while the JUnit example requires "mental backtracking". Another benefit of Truth over JUnit is the addition of useful default failure messages. For example: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">ImmutableSet<String> colors = ImmutableSet.of("red", "green", "blue", "yellow"); assertTrue(colors.contains("orange")); // JUnit assertThat(colors).contains("orange"); // Truth</pre> In this example, both assertions will fail, but JUnit will not provide a useful failure message. However, Truth will provide a clear and concise failure message: AssertionError: <[red, green, blue, yellow]> should have contained <orange> Truth already supports specialized assertions for most of the common JDK types (Objects, primitives, arrays, Strings, Classes, Comparables, Iterables, Collections, Lists, Sets, Maps, etc.), as well as some Guava types (Optionals). Additional support for other popular types is planned as well (Throwables, Iterators, Multimaps, UnsignedIntegers, UnsignedLongs, etc.). Truth is also user-extensible: you can easily write a Truth subject to make fluent assertions about your own custom types. By creating your own custom subject, both your assertion API and your failure messages can be domain-specific. Truth's goal is not to replace JUnit assertions, but to improve the readability of complex assertions and their failure messages. JUnit assertions and Truth assertions can (and often do) live side by side in tests. To get started with Truth, check out <a href="http://google.github.io/truth/">http://google.github.io/truth/</a> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Dori Reuveni and Kurt Alfred Kluever This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1TyLthjSGeELWVloxx2hI9l57pNRSN4UVAaNdB7H4Ctg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. As engineers, we spend most of our time reading existing code, rather than writing new code. Therefore, we must make sure we always write clean, readable code. The same goes for our tests; we need a way to clearly express our test assertions. Truth is an open source, fluent testing framework for Java designed to make your test assertions and failure messages more readable. The fluent API makes reading (and writing) test assertions much more natural, prose-like, and discoverable in your IDE via autocomplete. For example, compare how the following assertion reads with JUnit vs. Truth: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">assertEquals("March", monthMap.get(3)); // JUnit assertThat(monthMap).containsEntry(3, "March"); // Truth</pre> Both statements are asserting the same thing, but the assertion written with Truth can be easily read from left to right, while the JUnit example requires "mental backtracking". Another benefit of Truth over JUnit is the addition of useful default failure messages. For example: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">ImmutableSet<String> colors = ImmutableSet.of("red", "green", "blue", "yellow"); assertTrue(colors.contains("orange")); // JUnit assertThat(colors).contains("orange"); // Truth</pre> In this example, both assertions will fail, but JUnit will not provide a useful failure message. However, Truth will provide a clear and concise failure message: AssertionError: <[red, green, blue, yellow]> should have contained <orange> Truth already supports specialized assertions for most of the common JDK types (Objects, primitives, arrays, Strings, Classes, Comparables, Iterables, Collections, Lists, Sets, Maps, etc.), as well as some Guava types (Optionals). Additional support for other popular types is planned as well (Throwables, Iterators, Multimaps, UnsignedIntegers, UnsignedLongs, etc.). Truth is also user-extensible: you can easily write a Truth subject to make fluent assertions about your own custom types. By creating your own custom subject, both your assertion API and your failure messages can be domain-specific. Truth's goal is not to replace JUnit assertions, but to improve the readability of complex assertions and their failure messages. JUnit assertions and Truth assertions can (and often do) live side by side in tests. To get started with Truth, check out <a href="http://google.github.io/truth/">http://google.github.io/truth/</a> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Truth: a fluent assertion framework&url=https://testing.googleblog.com/2014/12/testing-on-toilet-truth-fluent.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/12/testing-on-toilet-truth-fluent.html#comments' style='font-weight: 500; text-decoration: underline;'>10 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/12/testing-on-toilet-truth-fluent.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Dori%20Reuveni' rel='tag'> Dori Reuveni </a> , <a class='label' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever' rel='tag'> Kurt Alfred Kluever </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='6330532562729532191' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/12/gtac-2014-wrap-up.html' itemprop='url' title='GTAC 2014 Wrap-up'> GTAC 2014 Wrap-up </a> </h2> <div class='post-header'> <div class='published'> Thursday, December 04, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee On October 28th and 29th, <a href="https://developers.google.com/google-test-automation-conference/2014/">GTAC 2014</a>, the eighth GTAC (Google Test Automation Conference), was held at the beautiful <a href="//www.google.com/about/careers/locations/seattle-kirkland/">Google Kirkland office</a>. The conference was completely packed with presenters and attendees from all over the world (Argentina, Australia, Canada, China, many European countries, India, Israel, Korea, New Zealand, Puerto Rico, Russia, Taiwan, and many US states), bringing with them a huge diversity of experiences. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9N0Wj4eoXstsSLqFQwabk46fo1vx5TxWpnrEimwA-diqEudhKkWg_g-SQzAio07RllM5RJ0HgaYmCYCZxBZPHKY6EniTrQ4oboiwQxDGgkausR0Hufn37A6VLz2ewbyTEpQsR/s1600/gtac-reception.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9N0Wj4eoXstsSLqFQwabk46fo1vx5TxWpnrEimwA-diqEudhKkWg_g-SQzAio07RllM5RJ0HgaYmCYCZxBZPHKY6EniTrQ4oboiwQxDGgkausR0Hufn37A6VLz2ewbyTEpQsR/s1600/gtac-reception.jpg" width="400" /></a></div> Speakers from numerous companies and universities (Adobe, American Express, Comcast, Dropbox, Facebook, FINRA, Google, HP, Medidata Solutions, Mozilla, Netflix, Orange, and University of Waterloo) spoke on a variety of interesting and cutting edge test automation topics. All of the <a href="https://developers.google.com/google-test-automation-conference/2014/presentations">slides and video recordings</a> are now available on the GTAC site. Photos will be available soon as well. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEibAvrBYTtK6aTIyG9Ye8sRZIqY1leHqA__PFS6VzscrSZGmmEQmD1T_aal7l1TPCPE9FJ3lRuss7mN5QevHC_nMPNL5RrnSzBkMfB-jIZmiXcNH9eyV3hi8OsNY9ZNET2oJ0vI/s1600/gtac-crowd.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEibAvrBYTtK6aTIyG9Ye8sRZIqY1leHqA__PFS6VzscrSZGmmEQmD1T_aal7l1TPCPE9FJ3lRuss7mN5QevHC_nMPNL5RrnSzBkMfB-jIZmiXcNH9eyV3hi8OsNY9ZNET2oJ0vI/s1600/gtac-crowd.jpg" width="400" /></a></div> This was our most popular GTAC to date, with over 1,500 applicants and almost 200 of those for speaking. About 250 people filled our venue to capacity, and the live stream had a peak of about 400 concurrent viewers with 4,700 playbacks during the event. And, there was plenty of interesting <a href="https://twitter.com/search?q=%23gtac2014">Twitter</a> and <a href="https://plus.google.com/u/0/explore/gtac2014">Google+</a> activity during the event. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg3PD1V8fNOwkqtc9bps1VdJEKeufodsvYCFsIYQBdnTz2YJnV2ERkz8bGymtDj5BR0zuniYNp_4TbCbt00XIqBbys3-y59-aCFZJeBcW4_Zn3vGkiOk_lTJbm2RgGEVNCbR6HQ/s1600/gtac-eat.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg3PD1V8fNOwkqtc9bps1VdJEKeufodsvYCFsIYQBdnTz2YJnV2ERkz8bGymtDj5BR0zuniYNp_4TbCbt00XIqBbys3-y59-aCFZJeBcW4_Zn3vGkiOk_lTJbm2RgGEVNCbR6HQ/s1600/gtac-eat.jpg" width="400" /></a></div> Our goal in hosting GTAC is to make the conference highly relevant and useful for, not only attendees, but the larger test engineering community as a whole. Our post-conference survey shows that we are close to achieving that goal: <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYZ0Dc45Z9sOrr0qyMYrtR8MaSkBv2-1s1t6V0Vrg_1IGCk7J49T5knwc5SQ-Mzi6vnBA-XQreDSgVdvh50Pbl26O58ScX0T-wDD7vQ_wEFHN3NnHBSzWQcTxj8oKUC3SkFBR9/s1600/gtac-stats.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="154" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYZ0Dc45Z9sOrr0qyMYrtR8MaSkBv2-1s1t6V0Vrg_1IGCk7J49T5knwc5SQ-Mzi6vnBA-XQreDSgVdvh50Pbl26O58ScX0T-wDD7vQ_wEFHN3NnHBSzWQcTxj8oKUC3SkFBR9/s1600/gtac-stats.png" width="640" /></a></div> If you have any suggestions on how we can improve, please comment on this post. Thank you to all the <a href="https://developers.google.com/google-test-automation-conference/2014/speakers">speakers</a>, attendees, and online viewers who made this a special event once again. To receive announcements about the next GTAC, subscribe to the <a href="http://googletesting.blogspot.com/">Google Testing Blog</a>. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee On October 28th and 29th, <a href="https://developers.google.com/google-test-automation-conference/2014/">GTAC 2014</a>, the eighth GTAC (Google Test Automation Conference), was held at the beautiful <a href="//www.google.com/about/careers/locations/seattle-kirkland/">Google Kirkland office</a>. The conference was completely packed with presenters and attendees from all over the world (Argentina, Australia, Canada, China, many European countries, India, Israel, Korea, New Zealand, Puerto Rico, Russia, Taiwan, and many US states), bringing with them a huge diversity of experiences. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9N0Wj4eoXstsSLqFQwabk46fo1vx5TxWpnrEimwA-diqEudhKkWg_g-SQzAio07RllM5RJ0HgaYmCYCZxBZPHKY6EniTrQ4oboiwQxDGgkausR0Hufn37A6VLz2ewbyTEpQsR/s1600/gtac-reception.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9N0Wj4eoXstsSLqFQwabk46fo1vx5TxWpnrEimwA-diqEudhKkWg_g-SQzAio07RllM5RJ0HgaYmCYCZxBZPHKY6EniTrQ4oboiwQxDGgkausR0Hufn37A6VLz2ewbyTEpQsR/s1600/gtac-reception.jpg" width="400" /></a></div> Speakers from numerous companies and universities (Adobe, American Express, Comcast, Dropbox, Facebook, FINRA, Google, HP, Medidata Solutions, Mozilla, Netflix, Orange, and University of Waterloo) spoke on a variety of interesting and cutting edge test automation topics. All of the <a href="https://developers.google.com/google-test-automation-conference/2014/presentations">slides and video recordings</a> are now available on the GTAC site. Photos will be available soon as well. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEibAvrBYTtK6aTIyG9Ye8sRZIqY1leHqA__PFS6VzscrSZGmmEQmD1T_aal7l1TPCPE9FJ3lRuss7mN5QevHC_nMPNL5RrnSzBkMfB-jIZmiXcNH9eyV3hi8OsNY9ZNET2oJ0vI/s1600/gtac-crowd.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEibAvrBYTtK6aTIyG9Ye8sRZIqY1leHqA__PFS6VzscrSZGmmEQmD1T_aal7l1TPCPE9FJ3lRuss7mN5QevHC_nMPNL5RrnSzBkMfB-jIZmiXcNH9eyV3hi8OsNY9ZNET2oJ0vI/s1600/gtac-crowd.jpg" width="400" /></a></div> This was our most popular GTAC to date, with over 1,500 applicants and almost 200 of those for speaking. About 250 people filled our venue to capacity, and the live stream had a peak of about 400 concurrent viewers with 4,700 playbacks during the event. And, there was plenty of interesting <a href="https://twitter.com/search?q=%23gtac2014">Twitter</a> and <a href="https://plus.google.com/u/0/explore/gtac2014">Google+</a> activity during the event. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg3PD1V8fNOwkqtc9bps1VdJEKeufodsvYCFsIYQBdnTz2YJnV2ERkz8bGymtDj5BR0zuniYNp_4TbCbt00XIqBbys3-y59-aCFZJeBcW4_Zn3vGkiOk_lTJbm2RgGEVNCbR6HQ/s1600/gtac-eat.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg3PD1V8fNOwkqtc9bps1VdJEKeufodsvYCFsIYQBdnTz2YJnV2ERkz8bGymtDj5BR0zuniYNp_4TbCbt00XIqBbys3-y59-aCFZJeBcW4_Zn3vGkiOk_lTJbm2RgGEVNCbR6HQ/s1600/gtac-eat.jpg" width="400" /></a></div> Our goal in hosting GTAC is to make the conference highly relevant and useful for, not only attendees, but the larger test engineering community as a whole. Our post-conference survey shows that we are close to achieving that goal: <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYZ0Dc45Z9sOrr0qyMYrtR8MaSkBv2-1s1t6V0Vrg_1IGCk7J49T5knwc5SQ-Mzi6vnBA-XQreDSgVdvh50Pbl26O58ScX0T-wDD7vQ_wEFHN3NnHBSzWQcTxj8oKUC3SkFBR9/s1600/gtac-stats.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="154" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYZ0Dc45Z9sOrr0qyMYrtR8MaSkBv2-1s1t6V0Vrg_1IGCk7J49T5knwc5SQ-Mzi6vnBA-XQreDSgVdvh50Pbl26O58ScX0T-wDD7vQ_wEFHN3NnHBSzWQcTxj8oKUC3SkFBR9/s1600/gtac-stats.png" width="640" /></a></div> If you have any suggestions on how we can improve, please comment on this post. Thank you to all the <a href="https://developers.google.com/google-test-automation-conference/2014/speakers">speakers</a>, attendees, and online viewers who made this a special event once again. To receive announcements about the next GTAC, subscribe to the <a href="http://googletesting.blogspot.com/">Google Testing Blog</a>. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2014 Wrap-up&url=https://testing.googleblog.com/2014/12/gtac-2014-wrap-up.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/12/gtac-2014-wrap-up.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/12/gtac-2014-wrap-up.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='9109027229271952901' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/11/protractor-angular-testing-made-easy.html' itemprop='url' title='Protractor: Angular testing made easy'> Protractor: Angular testing made easy </a> </h2> <div class='post-header'> <div class='published'> Sunday, November 30, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Hank Duan, Julie Ralph, and Arif Sukoco in Seattle Have you worked with WebDriver but been frustrated with all the waits needed for WebDriver to sync with the website, causing flakes and prolonged test times? If you are working with AngularJS apps, then Protractor is the right tool for you. Protractor (<a href="http://protractortest.org/">protractortest.org</a>) is an end-to-end test framework specifically for AngularJS apps. It was built by a team in Google and released to open source. Protractor is built on top of WebDriverJS and includes important improvements tailored for AngularJS apps. Here are some of Protractor’s key benefits: <ul style="line-height: 1.3em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>You don’t need to add waits or sleeps to your test. Protractor can communicate with your AngularJS app automatically and execute the next step in your test the moment the webpage finishes pending tasks, so you don’t have to worry about waiting for your test and webpage to sync. </li> <li>It supports Angular-specific locator strategies (e.g., binding, model, repeater) as well as native WebDriver locator strategies (e.g., ID, CSS selector, XPath). This allows you to test Angular-specific elements without any setup effort on your part. </li> <li>It is easy to set up page objects. Protractor does not execute WebDriver commands until an action is needed (e.g., get, sendKeys, click). This way you can set up page objects so tests can manipulate page elements without touching the HTML. </li> <li>It uses Jasmine, the framework you use to write AngularJS unit tests, and Javascript, the same language you use to write AngularJS apps.</li> </ul> Follow these simple steps, and in minutes, you will have you first Protractor test running: 1) Set up environment Install the command line tools ‘protractor’ and ‘webdriver-manager’ using npm: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">npm install -g protractor</pre> Start up an instance of a selenium server: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">webdriver-manager update & webdriver-manager start</pre> This downloads the necessary binary, and starts a new webdriver session listening on <a href="http://localhost:4444/">http://localhost:4444</a>. 2) Write your test <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// It is a good idea to use page objects to modularize your testing logic var angularHomepage = { nameInput : element(by.model('yourName')), greeting : element(by.binding('yourName')), get : function() { browser.get('index.html'); }, setName : function(name) { this.nameInput.sendKeys(name); } }; // Here we are using the Jasmine test framework // See http://jasmine.github.io/2.0/introduction.html for more details describe('angularjs homepage', function() { it('should greet the named user', function(){ angularHomepage.get(); angularHomepage.setName('Julie'); expect(angularHomepage.greeting.getText()). toEqual('Hello Julie!'); }); });</pre> 3) Write a Protractor configuration file to specify the environment under which you want your test to run: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">exports.config = { seleniumAddress: 'http://localhost:4444/wd/hub', specs: ['testFolder/*'], multiCapabilities: [{ 'browserName': 'chrome', // browser-specific tests specs: 'chromeTests/*' }, { 'browserName': 'firefox', // run tests in parallel shardTestFiles: true }], baseUrl: 'http://www.angularjs.org', };</pre> 4) Run the test: Start the test with the command: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">protractor conf.js</pre> The test output should be: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">1 test, 1 assertions, 0 failures</pre> If you want to learn more, here’s a full tutorial that highlights all of Protractor’s features: <a href="http://angular.github.io/protractor/#/tutorial">http://angular.github.io/protractor/#/tutorial </a> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Hank Duan, Julie Ralph, and Arif Sukoco in Seattle Have you worked with WebDriver but been frustrated with all the waits needed for WebDriver to sync with the website, causing flakes and prolonged test times? If you are working with AngularJS apps, then Protractor is the right tool for you. Protractor (<a href="http://protractortest.org/">protractortest.org</a>) is an end-to-end test framework specifically for AngularJS apps. It was built by a team in Google and released to open source. Protractor is built on top of WebDriverJS and includes important improvements tailored for AngularJS apps. Here are some of Protractor’s key benefits: <ul style="line-height: 1.3em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>You don’t need to add waits or sleeps to your test. Protractor can communicate with your AngularJS app automatically and execute the next step in your test the moment the webpage finishes pending tasks, so you don’t have to worry about waiting for your test and webpage to sync. </li> <li>It supports Angular-specific locator strategies (e.g., binding, model, repeater) as well as native WebDriver locator strategies (e.g., ID, CSS selector, XPath). This allows you to test Angular-specific elements without any setup effort on your part. </li> <li>It is easy to set up page objects. Protractor does not execute WebDriver commands until an action is needed (e.g., get, sendKeys, click). This way you can set up page objects so tests can manipulate page elements without touching the HTML. </li> <li>It uses Jasmine, the framework you use to write AngularJS unit tests, and Javascript, the same language you use to write AngularJS apps.</li> </ul> Follow these simple steps, and in minutes, you will have you first Protractor test running: 1) Set up environment Install the command line tools ‘protractor’ and ‘webdriver-manager’ using npm: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">npm install -g protractor</pre> Start up an instance of a selenium server: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">webdriver-manager update & webdriver-manager start</pre> This downloads the necessary binary, and starts a new webdriver session listening on <a href="http://localhost:4444/">http://localhost:4444</a>. 2) Write your test <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// It is a good idea to use page objects to modularize your testing logic var angularHomepage = { nameInput : element(by.model('yourName')), greeting : element(by.binding('yourName')), get : function() { browser.get('index.html'); }, setName : function(name) { this.nameInput.sendKeys(name); } }; // Here we are using the Jasmine test framework // See http://jasmine.github.io/2.0/introduction.html for more details describe('angularjs homepage', function() { it('should greet the named user', function(){ angularHomepage.get(); angularHomepage.setName('Julie'); expect(angularHomepage.greeting.getText()). toEqual('Hello Julie!'); }); });</pre> 3) Write a Protractor configuration file to specify the environment under which you want your test to run: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">exports.config = { seleniumAddress: 'http://localhost:4444/wd/hub', specs: ['testFolder/*'], multiCapabilities: [{ 'browserName': 'chrome', // browser-specific tests specs: 'chromeTests/*' }, { 'browserName': 'firefox', // run tests in parallel shardTestFiles: true }], baseUrl: 'http://www.angularjs.org', };</pre> 4) Run the test: Start the test with the command: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">protractor conf.js</pre> The test output should be: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">1 test, 1 assertions, 0 failures</pre> If you want to learn more, here’s a full tutorial that highlights all of Protractor’s features: <a href="http://angular.github.io/protractor/#/tutorial">http://angular.github.io/protractor/#/tutorial </a> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Protractor: Angular testing made easy&url=https://testing.googleblog.com/2014/11/protractor-angular-testing-made-easy.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/11/protractor-angular-testing-made-easy.html#comments' style='font-weight: 500; text-decoration: underline;'>1 comment</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/11/protractor-angular-testing-made-easy.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Arif%20Sukoco' rel='tag'> Arif Sukoco </a> , <a class='label' href='https://testing.googleblog.com/search/label/Hank%20Duan' rel='tag'> Hank Duan </a> , <a class='label' href='https://testing.googleblog.com/search/label/Julie%20Ralph' rel='tag'> Julie Ralph </a> </div> </div> </div> <div class='post' data-id='5562862972034441493' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/10/gtac-2014-is-this-week.html' itemprop='url' title='GTAC 2014 is this Week!'> GTAC 2014 is this Week! </a> </h2> <div class='post-header'> <div class='published'> Monday, October 27, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee The eighth GTAC commences on Tuesday at the Google Kirkland office. You can find the latest details on the conference at our site, including <a href="https://developers.google.com/google-test-automation-conference/2014/speakers">speaker profiles</a>. If you are watching remotely, we'll soon be updating the <a href="https://developers.google.com/google-test-automation-conference/2014/stream">live stream page</a> with the stream link and a Google Moderator link for remote Q&A. If you have been selected to attend or speak, be sure to note the <a href="https://developers.google.com/google-test-automation-conference/2014/travel">updated parking information</a>. Google visitors will use off-site parking and shuttles. We look forward to connecting with the greater testing community and sharing new advances and ideas. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee The eighth GTAC commences on Tuesday at the Google Kirkland office. You can find the latest details on the conference at our site, including <a href="https://developers.google.com/google-test-automation-conference/2014/speakers">speaker profiles</a>. If you are watching remotely, we'll soon be updating the <a href="https://developers.google.com/google-test-automation-conference/2014/stream">live stream page</a> with the stream link and a Google Moderator link for remote Q&A. If you have been selected to attend or speak, be sure to note the <a href="https://developers.google.com/google-test-automation-conference/2014/travel">updated parking information</a>. Google visitors will use off-site parking and shuttles. We look forward to connecting with the greater testing community and sharing new advances and ideas. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2014 is this Week!&url=https://testing.googleblog.com/2014/10/gtac-2014-is-this-week.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/10/gtac-2014-is-this-week.html#comments' style='font-weight: 500; text-decoration: underline;'>5 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/10/gtac-2014-is-this-week.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='3957960998516196391' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/10/testing-on-toilet-writing-descriptive.html' itemprop='url' title='Testing on the Toilet: Writing Descriptive Test Names'> Testing on the Toilet: Writing Descriptive Test Names </a> </h2> <div class='post-header'> <div class='published'> Thursday, October 16, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1UzvzMUkv0mpIFjrLqTcqvHFC5_1ycOCmrMVt-ubR2ec/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. How long does it take you to figure out what behavior is being tested in the following code? <pre style="background: rgb(255, 242, 204); border-color: rgb(123, 123, 123); border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void isUserLockedOut_invalidLogin() { authenticator.authenticate(username, invalidPassword); assertFalse(authenticator.isUserLockedOut(username)); authenticator.authenticate(username, invalidPassword); assertFalse(authenticator.isUserLockedOut(username)); authenticator.authenticate(username, invalidPassword); assertTrue(authenticator.isUserLockedOut(username)); }</pre> You probably had to read through every line of code (maybe more than once) and understand what each line is doing. But how long would it take you to figure out what behavior is being tested if the test had this name? isUserLockedOut_lockOutUserAfterThreeInvalidLoginAttempts You should now be able to understand what behavior is being tested by reading just the test name, and you don’t even need to read through the test body. The test name in the above code sample hints at the scenario being tested (“invalidLogin”), but it doesn’t actually say what the expected outcome is supposed to be, so you had to read through the code to figure it out. Putting both the scenario and the expected outcome in the test name has several other benefits: - If you want to know all the possible behaviors a class has, all you need to do is read through the test names in its test class, compared to spending minutes or hours digging through the test code or even the class itself trying to figure out its behavior. This can also be useful during code reviews since you can quickly tell if the tests cover all expected cases. - By giving tests more explicit names, it forces you to split up testing different behaviors into separate tests. Otherwise you may be tempted to dump assertions for different behaviors into one test, which over time can lead to tests that keep growing and become difficult to understand and maintain. - The exact behavior being tested might not always be clear from the test code. If the test name isn’t explicit about this, sometimes you might have to guess what the test is actually testing. - You can easily tell if some functionality isn’t being tested. If you don’t see a test name that describes the behavior you’re looking for, then you know the test doesn’t exist. - When a test fails, you can immediately see what functionality is broken without looking at the test’s source code. There are several common patterns for structuring the name of a test (one example is to name tests like an English sentence with “should” in the name, e.g., shouldLockOutUserAfterThreeInvalidLoginAttempts). Whichever pattern you use, the same advice still applies: Make sure test names contain both the scenario being tested and the expected outcome. Sometimes just specifying the name of the method under test may be enough, especially if the method is simple and has only a single behavior that is obvious from its name. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1UzvzMUkv0mpIFjrLqTcqvHFC5_1ycOCmrMVt-ubR2ec/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. How long does it take you to figure out what behavior is being tested in the following code? <pre style="background: rgb(255, 242, 204); border-color: rgb(123, 123, 123); border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void isUserLockedOut_invalidLogin() { authenticator.authenticate(username, invalidPassword); assertFalse(authenticator.isUserLockedOut(username)); authenticator.authenticate(username, invalidPassword); assertFalse(authenticator.isUserLockedOut(username)); authenticator.authenticate(username, invalidPassword); assertTrue(authenticator.isUserLockedOut(username)); }</pre> You probably had to read through every line of code (maybe more than once) and understand what each line is doing. But how long would it take you to figure out what behavior is being tested if the test had this name? isUserLockedOut_lockOutUserAfterThreeInvalidLoginAttempts You should now be able to understand what behavior is being tested by reading just the test name, and you don’t even need to read through the test body. The test name in the above code sample hints at the scenario being tested (“invalidLogin”), but it doesn’t actually say what the expected outcome is supposed to be, so you had to read through the code to figure it out. Putting both the scenario and the expected outcome in the test name has several other benefits: - If you want to know all the possible behaviors a class has, all you need to do is read through the test names in its test class, compared to spending minutes or hours digging through the test code or even the class itself trying to figure out its behavior. This can also be useful during code reviews since you can quickly tell if the tests cover all expected cases. - By giving tests more explicit names, it forces you to split up testing different behaviors into separate tests. Otherwise you may be tempted to dump assertions for different behaviors into one test, which over time can lead to tests that keep growing and become difficult to understand and maintain. - The exact behavior being tested might not always be clear from the test code. If the test name isn’t explicit about this, sometimes you might have to guess what the test is actually testing. - You can easily tell if some functionality isn’t being tested. If you don’t see a test name that describes the behavior you’re looking for, then you know the test doesn’t exist. - When a test fails, you can immediately see what functionality is broken without looking at the test’s source code. There are several common patterns for structuring the name of a test (one example is to name tests like an English sentence with “should” in the name, e.g., shouldLockOutUserAfterThreeInvalidLoginAttempts). Whichever pattern you use, the same advice still applies: Make sure test names contain both the scenario being tested and the expected outcome. Sometimes just specifying the name of the method under test may be enough, especially if the method is simple and has only a single behavior that is obvious from its name. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Writing Descriptive Test Names&url=https://testing.googleblog.com/2014/10/testing-on-toilet-writing-descriptive.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/10/testing-on-toilet-writing-descriptive.html#comments' style='font-weight: 500; text-decoration: underline;'>4 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/10/testing-on-toilet-writing-descriptive.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='2984740049089789201' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/09/announcing-gtac-2014-agenda.html' itemprop='url' title='Announcing the GTAC 2014 Agenda'> Announcing the GTAC 2014 Agenda </a> </h2> <div class='post-header'> <div class='published'> Tuesday, September 30, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee We have completed selection and confirmation of all speakers and attendees for GTAC 2014. You can find the detailed agenda at:   <a href="http://developers.google.com/gtac/2014/schedule">developers.google.com/gtac/2014/schedule</a> Thank you to all who submitted proposals! It was very hard to make selections from so many fantastic submissions. There was a tremendous amount of interest in GTAC this year with over 1,500 applicants (up from 533 last year) and 194 of those for speaking (up from 88 last year). Unfortunately, our venue only seats 250. However, don’t despair if you did not receive an invitation. Just like last year, anyone can join us via <a href="https://developers.google.com/google-test-automation-conference/2014/stream">YouTube live streaming</a>. We’ll also be setting up Google Moderator, so remote attendees can get involved in Q&A after each talk. Information about live streaming, Moderator, and other details will be posted on the GTAC site soon and announced here. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee We have completed selection and confirmation of all speakers and attendees for GTAC 2014. You can find the detailed agenda at:   <a href="http://developers.google.com/gtac/2014/schedule">developers.google.com/gtac/2014/schedule</a> Thank you to all who submitted proposals! It was very hard to make selections from so many fantastic submissions. There was a tremendous amount of interest in GTAC this year with over 1,500 applicants (up from 533 last year) and 194 of those for speaking (up from 88 last year). Unfortunately, our venue only seats 250. However, don’t despair if you did not receive an invitation. Just like last year, anyone can join us via <a href="https://developers.google.com/google-test-automation-conference/2014/stream">YouTube live streaming</a>. We’ll also be setting up Google Moderator, so remote attendees can get involved in Q&A after each talk. Information about live streaming, Moderator, and other details will be posted on the GTAC site soon and announced here. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Announcing the GTAC 2014 Agenda&url=https://testing.googleblog.com/2014/09/announcing-gtac-2014-agenda.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/09/announcing-gtac-2014-agenda.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/09/announcing-gtac-2014-agenda.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='4273326115277895932' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/09/chrome-firefox-webrtc-interop-test-pt-2.html' itemprop='url' title='Chrome - Firefox WebRTC Interop Test - Pt 2'> Chrome - Firefox WebRTC Interop Test - Pt 2 </a> </h2> <div class='post-header'> <div class='published'> Tuesday, September 09, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Patrik Höglund This is the second in a series of articles about Chrome’s WebRTC Interop Test. <a href="http://googletesting.blogspot.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html">See the first</a>. In the previous blog post we managed to write an automated test which got a WebRTC call between Firefox and Chrome to run. But how do we verify that the call actually worked? <h3> Verifying the Call</h3> Now we can launch the two browsers, but how do we figure out the whether the call actually worked? If you try opening two <a href="http://apprtc.appspot.com/">apprtc.appspot.com</a> tabs in the same room, you will notice the video feeds flip over using a CSS transform, your local video is relegated to a small frame and a new big video feed with the remote video shows up. For the first version of the test, I just looked at the page in the Chrome debugger and looked for some reliable signal. As it turns out, the remoteVideo.style.opacity property will go from 0 to 1 when the call goes up and from 1 to 0 when it goes down. Since we can execute arbitrary JavaScript in the Chrome tab from the test, we can simply implement the check like this: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool WaitForCallToComeUp(content::WebContents* tab_contents) { // Apprtc will set remoteVideo.style.opacity to 1 when the call comes up. std::string javascript = "window.domAutomationController.send(remoteVideo.style.opacity)"; return test::PollingWaitUntil(javascript, "1", tab_contents); } </pre> <h3> Verifying Video is Playing</h3> So getting a call up is good, but what if there is a bug where Firefox and Chrome cannot send correct video streams to each other? To check that, we needed to step up our game a bit. We decided to use our <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/test/data/webrtc/video_detector.js&q=video_dete&sq=package:chromium&l=1">existing video detector</a>, which looks at a video element and determines if the pixels are changing. This is a very basic check, but it’s better than nothing. To do this, we simply evaluate the .js file’s JavaScript in the context of the Chrome tab, making the functions in the file available to us. The implementation then becomes <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool DetectRemoteVideoPlaying(content::WebContents* tab_contents) { if (!EvalInJavascriptFile(tab_contents, GetSourceDir().Append( FILE_PATH_LITERAL( "chrome/test/data/webrtc/test_functions.js")))) return false; if (!EvalInJavascriptFile(tab_contents, GetSourceDir().Append( FILE_PATH_LITERAL( "chrome/test/data/webrtc/video_detector.js")))) return false; // The remote video tag is called remoteVideo in the AppRTC code. StartDetectingVideo(tab_contents, "remoteVideo"); WaitForVideoToPlay(tab_contents); return true; }</pre> where StartDetectingVideo and WaitForVideoToPlay call the corresponding JavaScript methods in <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/test/data/webrtc/video_detector.js&q=video_dete&sq=package:chromium&l=61">video_detector.js</a>. If the video feed is frozen and unchanging, the test will time out and fail. <h3> What to Send in the Call</h3> Now we can get a call up between the browsers and detect if video is playing. But what video should we send? For chrome, we have a convenient --use-fake-device-for-media-stream flag that will make Chrome pretend there’s a webcam and present a generated video feed (which is a spinning green ball with a timestamp). This turned out to be useful since Firefox and Chrome cannot acquire the same camera at the same time, so if we didn’t use the fake device we would have two webcams plugged into the bots executing the tests! Bots running in Chrome’s <a href="http://build.chromium.org/">regular test infrastructure</a> do not have either software or hardware webcams plugged into them, so this test must run on bots with webcams for Firefox to be able to acquire a camera. Fortunately, we have that in the <a href="http://build.chromium.org/p/chromium.webrtc/waterfall">WebRTC waterfalls</a> in order to test that we can actually acquire hardware webcams on all platforms. We also added a check to just succeed the test when there’s no real webcam on the system since we don’t want it to fail when a dev runs it on a machine without a webcam: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">if (!HasWebcamOnSystem()) return; </pre> It would of course be better if Firefox had a similar fake device, but to my knowledge it doesn’t. <h3> Downloading all Code and Components </h3> Now we have all we need to run the test and have it verify something useful. We just have the hard part left: how do we actually download all the resources we need to run this test? Recall that this is actually a three-way integration test between Chrome, Firefox and AppRTC, which require the following: <ul style="line-height: 1.0em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>The AppEngine SDK in order to bring up the local AppRTC instance, </li> <li>The AppRTC code itself, </li> <li>Chrome (already present in the checkout), and </li> <li>Firefox nightly.</li> </ul> While developing the test, I initially just hand-downloaded these and installed and hard-coded the paths. This is a very bad idea in the long run. Recall that the Chromium infrastructure is comprised of <a href="http://build.chromium.org/">thousands and thousands of machines</a>, and while this test will only run on perhaps 5 at a time due to its webcam requirements, we don’t want manual maintenance work whenever we replace a machine. And for that matter, we definitely don’t want to download a new Firefox by hand every night and put it on the right location on the bots! So how do we automate this? Downloading the AppEngine SDK First, let’s start with the easy part. We don’t really care if the AppEngine SDK is up-to-date, so a relatively stale version is fine. We could have the test download it from the <a href="https://developers.google.com/appengine/downloads">authoritative source</a>, but that’s a bad idea for a couple reasons. First, it updates outside our control. Second, there could be anti-robot measures on the page. Third, the download will likely be unreliable and fail the test occasionally. The way we solved this was to upload a copy of the SDK to a <a href="https://cloud.google.com/products/cloud-storage/?utm_source=google&utm_medium=cpc&utm_campaign=cloudstorage-search">Google storage</a> bucket under our control and download it using the <a href="http://dev.chromium.org/developers/how-tos/depottools">depot_tools</a> script download_from_google_storage.py. This is a lot more reliable than an external website and will not download the SDK if we already have the right version on the bot. Downloading the AppRTC Code This code is on GitHub. Experience has shown that git clone commands run against GitHub will fail every now and then, and fail the test. We could either write some retry mechanism, but we have found it’s better to simply mirror the git repository in Chromium’s internal mirrors, which are closer to our bots and thereby more reliable from our perspective. The pull is done by a <a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/DEPS?revision=262390&view=markup">Chromium DEPS file</a> (which is Chromium’s dependency provisioning framework). Downloading Firefox It turns out that Firefox supplies handy libraries for this task. We’re using mozdownload <a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/download_firefox_nightly.py?revision=241269&view=markup">in this script</a> in order to download the Firefox nightly build. Unfortunately this <a href="https://code.google.com/p/chromium/issues/detail?id=360516">fails every now and then</a> so we would like to have some retry mechanism, or we could write some mechanism to “mirror” the Firefox nightly build in some location we control. <h3> Putting it Together</h3> With that, we have everything we need to deploy the test. You can see the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_apprtc_browsertest.cc&q=chrome_webrtc_app&sq=package:chromium&l=297">final code here</a>. The provisioning code above was put into a separate “<a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/">.gclient solution</a>” so that regular Chrome devs and bots are not burdened with downloading hundreds of megs of SDKs and code that they will not use. When this test runs, you will first see a Chrome browser pop up, which will ensure the local apprtc instance is up. Then a Firefox browser will pop up. They will each acquire the fake device and real camera, respectively, and after a short delay the AppRTC call will come up, proving that video interop is working. This is a complicated and expensive test, but we believe it is worth it to keep the main interop case under automation this way, especially as the spec evolves and the browsers are in varying states of implementation. Future Work <ul style="line-height: 1.0em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Also run on Windows/Mac. </li> <li>Also test Opera. </li> <li>Interop between Chrome/Firefox mobile and desktop browsers. </li> <li>Also ensure audio is playing. </li> <li>Measure bandwidth stats, video quality, etc.</li> </ul> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Patrik Höglund This is the second in a series of articles about Chrome’s WebRTC Interop Test. <a href="http://googletesting.blogspot.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html">See the first</a>. In the previous blog post we managed to write an automated test which got a WebRTC call between Firefox and Chrome to run. But how do we verify that the call actually worked? <h3> Verifying the Call</h3> Now we can launch the two browsers, but how do we figure out the whether the call actually worked? If you try opening two <a href="http://apprtc.appspot.com/">apprtc.appspot.com</a> tabs in the same room, you will notice the video feeds flip over using a CSS transform, your local video is relegated to a small frame and a new big video feed with the remote video shows up. For the first version of the test, I just looked at the page in the Chrome debugger and looked for some reliable signal. As it turns out, the remoteVideo.style.opacity property will go from 0 to 1 when the call goes up and from 1 to 0 when it goes down. Since we can execute arbitrary JavaScript in the Chrome tab from the test, we can simply implement the check like this: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool WaitForCallToComeUp(content::WebContents* tab_contents) { // Apprtc will set remoteVideo.style.opacity to 1 when the call comes up. std::string javascript = "window.domAutomationController.send(remoteVideo.style.opacity)"; return test::PollingWaitUntil(javascript, "1", tab_contents); } </pre> <h3> Verifying Video is Playing</h3> So getting a call up is good, but what if there is a bug where Firefox and Chrome cannot send correct video streams to each other? To check that, we needed to step up our game a bit. We decided to use our <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/test/data/webrtc/video_detector.js&q=video_dete&sq=package:chromium&l=1">existing video detector</a>, which looks at a video element and determines if the pixels are changing. This is a very basic check, but it’s better than nothing. To do this, we simply evaluate the .js file’s JavaScript in the context of the Chrome tab, making the functions in the file available to us. The implementation then becomes <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool DetectRemoteVideoPlaying(content::WebContents* tab_contents) { if (!EvalInJavascriptFile(tab_contents, GetSourceDir().Append( FILE_PATH_LITERAL( "chrome/test/data/webrtc/test_functions.js")))) return false; if (!EvalInJavascriptFile(tab_contents, GetSourceDir().Append( FILE_PATH_LITERAL( "chrome/test/data/webrtc/video_detector.js")))) return false; // The remote video tag is called remoteVideo in the AppRTC code. StartDetectingVideo(tab_contents, "remoteVideo"); WaitForVideoToPlay(tab_contents); return true; }</pre> where StartDetectingVideo and WaitForVideoToPlay call the corresponding JavaScript methods in <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/test/data/webrtc/video_detector.js&q=video_dete&sq=package:chromium&l=61">video_detector.js</a>. If the video feed is frozen and unchanging, the test will time out and fail. <h3> What to Send in the Call</h3> Now we can get a call up between the browsers and detect if video is playing. But what video should we send? For chrome, we have a convenient --use-fake-device-for-media-stream flag that will make Chrome pretend there’s a webcam and present a generated video feed (which is a spinning green ball with a timestamp). This turned out to be useful since Firefox and Chrome cannot acquire the same camera at the same time, so if we didn’t use the fake device we would have two webcams plugged into the bots executing the tests! Bots running in Chrome’s <a href="http://build.chromium.org/">regular test infrastructure</a> do not have either software or hardware webcams plugged into them, so this test must run on bots with webcams for Firefox to be able to acquire a camera. Fortunately, we have that in the <a href="http://build.chromium.org/p/chromium.webrtc/waterfall">WebRTC waterfalls</a> in order to test that we can actually acquire hardware webcams on all platforms. We also added a check to just succeed the test when there’s no real webcam on the system since we don’t want it to fail when a dev runs it on a machine without a webcam: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">if (!HasWebcamOnSystem()) return; </pre> It would of course be better if Firefox had a similar fake device, but to my knowledge it doesn’t. <h3> Downloading all Code and Components </h3> Now we have all we need to run the test and have it verify something useful. We just have the hard part left: how do we actually download all the resources we need to run this test? Recall that this is actually a three-way integration test between Chrome, Firefox and AppRTC, which require the following: <ul style="line-height: 1.0em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>The AppEngine SDK in order to bring up the local AppRTC instance, </li> <li>The AppRTC code itself, </li> <li>Chrome (already present in the checkout), and </li> <li>Firefox nightly.</li> </ul> While developing the test, I initially just hand-downloaded these and installed and hard-coded the paths. This is a very bad idea in the long run. Recall that the Chromium infrastructure is comprised of <a href="http://build.chromium.org/">thousands and thousands of machines</a>, and while this test will only run on perhaps 5 at a time due to its webcam requirements, we don’t want manual maintenance work whenever we replace a machine. And for that matter, we definitely don’t want to download a new Firefox by hand every night and put it on the right location on the bots! So how do we automate this? Downloading the AppEngine SDK First, let’s start with the easy part. We don’t really care if the AppEngine SDK is up-to-date, so a relatively stale version is fine. We could have the test download it from the <a href="https://developers.google.com/appengine/downloads">authoritative source</a>, but that’s a bad idea for a couple reasons. First, it updates outside our control. Second, there could be anti-robot measures on the page. Third, the download will likely be unreliable and fail the test occasionally. The way we solved this was to upload a copy of the SDK to a <a href="https://cloud.google.com/products/cloud-storage/?utm_source=google&utm_medium=cpc&utm_campaign=cloudstorage-search">Google storage</a> bucket under our control and download it using the <a href="http://dev.chromium.org/developers/how-tos/depottools">depot_tools</a> script download_from_google_storage.py. This is a lot more reliable than an external website and will not download the SDK if we already have the right version on the bot. Downloading the AppRTC Code This code is on GitHub. Experience has shown that git clone commands run against GitHub will fail every now and then, and fail the test. We could either write some retry mechanism, but we have found it’s better to simply mirror the git repository in Chromium’s internal mirrors, which are closer to our bots and thereby more reliable from our perspective. The pull is done by a <a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/DEPS?revision=262390&view=markup">Chromium DEPS file</a> (which is Chromium’s dependency provisioning framework). Downloading Firefox It turns out that Firefox supplies handy libraries for this task. We’re using mozdownload <a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/download_firefox_nightly.py?revision=241269&view=markup">in this script</a> in order to download the Firefox nightly build. Unfortunately this <a href="https://code.google.com/p/chromium/issues/detail?id=360516">fails every now and then</a> so we would like to have some retry mechanism, or we could write some mechanism to “mirror” the Firefox nightly build in some location we control. <h3> Putting it Together</h3> With that, we have everything we need to deploy the test. You can see the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_apprtc_browsertest.cc&q=chrome_webrtc_app&sq=package:chromium&l=297">final code here</a>. The provisioning code above was put into a separate “<a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/">.gclient solution</a>” so that regular Chrome devs and bots are not burdened with downloading hundreds of megs of SDKs and code that they will not use. When this test runs, you will first see a Chrome browser pop up, which will ensure the local apprtc instance is up. Then a Firefox browser will pop up. They will each acquire the fake device and real camera, respectively, and after a short delay the AppRTC call will come up, proving that video interop is working. This is a complicated and expensive test, but we believe it is worth it to keep the main interop case under automation this way, especially as the spec evolves and the browsers are in varying states of implementation. Future Work <ul style="line-height: 1.0em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Also run on Windows/Mac. </li> <li>Also test Opera. </li> <li>Interop between Chrome/Firefox mobile and desktop browsers. </li> <li>Also ensure audio is playing. </li> <li>Measure bandwidth stats, video quality, etc.</li> </ul> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Chrome - Firefox WebRTC Interop Test - Pt 2&url=https://testing.googleblog.com/2014/09/chrome-firefox-webrtc-interop-test-pt-2.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/09/chrome-firefox-webrtc-interop-test-pt-2.html#comments' style='font-weight: 500; text-decoration: underline;'>3 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/09/chrome-firefox-webrtc-interop-test-pt-2.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund' rel='tag'> Patrik Höglund </a> </div> </div> </div> <div class='post' data-id='2586091485190798502' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html' itemprop='url' title='Chrome - Firefox WebRTC Interop Test - Pt 1'> Chrome - Firefox WebRTC Interop Test - Pt 1 </a> </h2> <div class='post-header'> <div class='published'> Tuesday, August 26, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Patrik Höglund <a href="http://www.webrtc.org/">WebRTC</a> enables real time peer-to-peer video and voice transfer in the browser, making it possible to build, among other things, a <a href="https://apprtc.appspot.com/">working video chat</a> with a <a href="https://github.com/GoogleChrome/webrtc/tree/master/samples/web/content/apprtc">small amount of Python and JavaScript</a>. As a web standard, it has several unusual properties which makes it hard to test. A regular web standard generally accepts HTML text and yields a bitmap as output (what you see in the browser). For WebRTC, we have real-time RTP media streams on one side being sent to another WebRTC-enabled endpoint. These RTP packets have been jumping across NAT, through firewalls and perhaps through TURN servers to deliver hopefully stutter-free and low latency media. WebRTC is probably the only web standard in which we need to test direct communication between Chrome and other browsers. Remember, WebRTC builds on peer-to-peer technology, which means we talk directly between browsers rather than through a server. Chrome, Firefox and Opera have announced support for WebRTC so far. To test interoperability, we set out to build an automated test to ensure that Chrome and Firefox can get a call up. This article describes how we implemented such a test and the tradeoffs we made along the way. <h3> Calling in WebRTC </h3> Setting up a WebRTC call requires passing SDP blobs over a signaling connection. These blobs contain information on the capabilities of the endpoint, such as what media formats it supports and what preferences it has (for instance, perhaps the endpoint has VP8 decoding hardware, which means the endpoint will handle VP8 more efficiently than, say, H.264). By sending these blobs the endpoints can agree on what media format they will be sending between themselves and how to traverse the network between them. Once that is done, the browsers will talk directly to each other, and nothing gets sent over the signaling connection. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjPSI18tUi-P0U03LzJcP6bbk8PSB-5fXLi-HSPGJ-1zexNRpVxKREigxle-R8eAQLxxzAflu-LF5XLVZyFBjFD7u93CVnO7pDKTfZq_xU9DuiVIRhbsluyq-L9WMzIghB1R_bX/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjPSI18tUi-P0U03LzJcP6bbk8PSB-5fXLi-HSPGJ-1zexNRpVxKREigxle-R8eAQLxxzAflu-LF5XLVZyFBjFD7u93CVnO7pDKTfZq_xU9DuiVIRhbsluyq-L9WMzIghB1R_bX/s1600/image00.png" width="500" /></a></div> <div style="text-align: center;"> Figure 1. Signaling and media connections.</div> How these blobs are sent is up to the application. Usually the browsers connect to some server which mediates the connection between the browsers, for instance by using a contact list or a room number. The <a href="https://apprtc.appspot.com/">AppRTC reference application</a> uses room numbers to pair up browsers and sends the SDP blobs from the browsers through the AppRTC server. <h3> Test Design</h3> Instead of designing a new signaling solution from scratch, we chose to use the AppRTC application we already had. This has the additional benefit of testing the AppRTC code, which we are also maintaining. We could also have used the small <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/libjingle/libjingle.gyp&q=peerconnection_server&sq=package:chromium&type=cs&l=289">peerconnection_server</a> binary and some JavaScript, which would give us additional flexibility in what to test. We chose to go with AppRTC since it effectively implements the signaling for us, leading to much less test code. We assumed we would be able to get hold of the latest nightly Firefox and be able to launch that with a given URL. For the Chrome side, we assumed we would be running in a browser test, i.e. on a complete Chrome with some test scaffolding around it. For the first sketch of the test, we imagined just connecting the browsers to the live <a href="http://apprtc.appspot.com/">apprtc.appspot.com</a> with some random room number. If the call got established, we would be able to look at the remote video feed on the Chrome side and verify that video was playing (for instance using the <a href="http://googlechrome.github.io/webrtc/samples/web/content/getusermedia-canvas">video+canvas grab trick</a>). Furthermore, we could verify that audio was playing, for instance by using <a href="http://googlechrome.github.io/webrtc/samples/web/content/getusermedia-volume">WebRTC getStats</a> to measure the audio track energy level. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjGVzL0DcoTmuyYSQdoaMzOTbBqr2Si69Hv8NoDwRc6rHYQ4_PTNgmYcRHcoe78owHgH50kkA6wD7Ec3E0GsUARSbEvVx34zTXOyQVObb4DzlD41hRh80OzAuNXhXZYTD-Iy10F/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjGVzL0DcoTmuyYSQdoaMzOTbBqr2Si69Hv8NoDwRc6rHYQ4_PTNgmYcRHcoe78owHgH50kkA6wD7Ec3E0GsUARSbEvVx34zTXOyQVObb4DzlD41hRh80OzAuNXhXZYTD-Iy10F/s1600/image01.png" width="500" /></a></div> <div style="text-align: center;"> Figure 2. Basic test design.</div> However, since we like tests to be <a href="http://googletesting.blogspot.se/2012/10/hermetic-servers.html">hermetic</a>, this isn’t a good design. I can see several problems. For example, if the network between us and AppRTC is unreliable. Also, what if someone has occupied myroomid? If that were the case, the test would fail and we would be none the wiser. So to make this thing work, we would have to find some way to bring up the AppRTC instance on localhost to make our test hermetic. <h3> Bringing up AppRTC on localhost</h3> AppRTC is a <a href="https://developers.google.com/appengine/">Google App Engine application</a>. As this <a href="https://developers.google.com/appengine/docs/python/gettingstartedpython27/helloworld">hello world example</a> demonstrates, one can test applications locally with <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">google_appengine/dev_appserver.py apprtc_code/</pre> So why not just call this from our test? It turns out we need to solve some complicated problems first, like how to ensure the AppEngine SDK and the AppRTC code is actually available on the executing machine, but we’ll get to that later. Let’s assume for now that stuff is just available. We can now write the browser test code to launch the local instance: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool LaunchApprtcInstanceOnLocalhost() // ... Figure out locations of SDK and apprtc code ... CommandLine command_line(CommandLine::NO_PROGRAM); EXPECT_TRUE(GetPythonCommand(&command_line)); command_line.AppendArgPath(appengine_dev_appserver); command_line.AppendArgPath(apprtc_dir); command_line.AppendArg("--port=9999"); command_line.AppendArg("--admin_port=9998"); command_line.AppendArg("--skip_sdk_update_check"); VLOG(1) << "Running " << command_line.GetCommandLineString(); return base::LaunchProcess(command_line, base::LaunchOptions(), &dev_appserver_); } </pre> That’s pretty straightforward <a href="#foot1">[1]</a>. <h3> Figuring out Whether the Local Server is Up </h3> Then we ran into a very typical test problem. So we have the code to get the server up, and launching the two browsers to connect to http://localhost:9999?r=some_room is easy. But how do we know when to connect? When I first ran the test, it would work sometimes and sometimes not depending on if the server had time to get up. It’s tempting in these situations to just add a sleep to give the server time to get up. Don’t do that. That will result in a test that is flaky and/or slow. In these situations we need to identify what we’re really waiting for. We could probably monitor the stdout of the dev_appserver.py and look for some message that says “Server is up!” or equivalent. However, we’re really waiting for the server to be able to serve web pages, and since we have two browsers that are really good at connecting to servers, why not use them? Consider this code. <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool LocalApprtcInstanceIsUp() { // Load the admin page and see if we manage to load it right. ui_test_utils::NavigateToURL(browser(), GURL("localhost:9998")); content::WebContents* tab_contents = browser()->tab_strip_model()->GetActiveWebContents(); std::string javascript = "window.domAutomationController.send(document.title)"; std::string result; if (!content::ExecuteScriptAndExtractString(tab_contents, javascript, &result)) return false; return result == kTitlePageOfAppEngineAdminPage; } </pre> Here we ask Chrome to load the AppEngine admin page for the local server (we set the admin port to 9998 earlier, remember?) and ask it what its title is. If that title is “Instances”, the admin page has been displayed, and the server must be up. If the server isn’t up, Chrome will fail to load the page and the title will be something like “localhost:9999 is not available”. Then, we can just do this from the test: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">while (!LocalApprtcInstanceIsUp()) VLOG(1) << "Waiting for AppRTC to come up..."; </pre> If the server never comes up, for whatever reason, the test will just time out in that loop. If it comes up we can safely proceed with the rest of test. <h3> Launching the Browsers </h3> A browser window launches itself as a part of every Chromium browser test. It’s also easy for the test to control the command line switches the browser will run under. We have less control over the Firefox browser since it is the “foreign” browser in this test, but we can still pass command-line options to it when we invoke the Firefox process. To make this easier, Mozilla provides a Python library called mozrunner. Using that we can set up a <a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/run_firefox_webrtc.py?revision=240895&view=markup">launcher python script</a> we can invoke from the test: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">from mozprofile import profile from mozrunner import runner WEBRTC_PREFERENCES = { 'media.navigator.permission.disabled': True, } def main(): # Set up flags, handle SIGTERM, etc # ... firefox_profile = profile.FirefoxProfile(preferences=WEBRTC_PREFERENCES) firefox_runner = runner.FirefoxRunner( profile=firefox_profile, binary=options.binary, cmdargs=[options.webpage]) firefox_runner.start()</pre> Notice that we need to pass special preferences to make Firefox accept the getUserMedia prompt. Otherwise, the test would get stuck on the prompt and we would be unable to set up a call. Alternatively, we could employ some kind of clickbot to click “Allow” on the prompt when it pops up, but that is way harder to set up. Without going into too much detail, the code for launching the browsers becomes <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">GURL room_url = GURL(base::StringPrintf("http://localhost:9999?r=room_%d", base::RandInt(0, 65536))); content::WebContents* chrome_tab = OpenPageAndAcceptUserMedia(room_url); ASSERT_TRUE(LaunchFirefoxWithUrl(room_url));</pre> Where LaunchFirefoxWithUrl essentially runs this: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">run_firefox_webrtc.py --binary /path/to/firefox --webpage http://localhost::9999?r=my_room</pre> Now we can launch the two browsers. Next time we will look at how we actually verify that the call worked, and how we actually download all resources needed by the test in a maintainable and automated manner. Stay tuned! <hr /> 1 The explicit ports are because the default ports collided on the bots we were running on, and the --skip_sdk_update_check was because the SDK stopped and asked us something if there was an update. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Patrik Höglund <a href="http://www.webrtc.org/">WebRTC</a> enables real time peer-to-peer video and voice transfer in the browser, making it possible to build, among other things, a <a href="https://apprtc.appspot.com/">working video chat</a> with a <a href="https://github.com/GoogleChrome/webrtc/tree/master/samples/web/content/apprtc">small amount of Python and JavaScript</a>. As a web standard, it has several unusual properties which makes it hard to test. A regular web standard generally accepts HTML text and yields a bitmap as output (what you see in the browser). For WebRTC, we have real-time RTP media streams on one side being sent to another WebRTC-enabled endpoint. These RTP packets have been jumping across NAT, through firewalls and perhaps through TURN servers to deliver hopefully stutter-free and low latency media. WebRTC is probably the only web standard in which we need to test direct communication between Chrome and other browsers. Remember, WebRTC builds on peer-to-peer technology, which means we talk directly between browsers rather than through a server. Chrome, Firefox and Opera have announced support for WebRTC so far. To test interoperability, we set out to build an automated test to ensure that Chrome and Firefox can get a call up. This article describes how we implemented such a test and the tradeoffs we made along the way. <h3> Calling in WebRTC </h3> Setting up a WebRTC call requires passing SDP blobs over a signaling connection. These blobs contain information on the capabilities of the endpoint, such as what media formats it supports and what preferences it has (for instance, perhaps the endpoint has VP8 decoding hardware, which means the endpoint will handle VP8 more efficiently than, say, H.264). By sending these blobs the endpoints can agree on what media format they will be sending between themselves and how to traverse the network between them. Once that is done, the browsers will talk directly to each other, and nothing gets sent over the signaling connection. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjPSI18tUi-P0U03LzJcP6bbk8PSB-5fXLi-HSPGJ-1zexNRpVxKREigxle-R8eAQLxxzAflu-LF5XLVZyFBjFD7u93CVnO7pDKTfZq_xU9DuiVIRhbsluyq-L9WMzIghB1R_bX/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjPSI18tUi-P0U03LzJcP6bbk8PSB-5fXLi-HSPGJ-1zexNRpVxKREigxle-R8eAQLxxzAflu-LF5XLVZyFBjFD7u93CVnO7pDKTfZq_xU9DuiVIRhbsluyq-L9WMzIghB1R_bX/s1600/image00.png" width="500" /></a></div> <div style="text-align: center;"> Figure 1. Signaling and media connections.</div> How these blobs are sent is up to the application. Usually the browsers connect to some server which mediates the connection between the browsers, for instance by using a contact list or a room number. The <a href="https://apprtc.appspot.com/">AppRTC reference application</a> uses room numbers to pair up browsers and sends the SDP blobs from the browsers through the AppRTC server. <h3> Test Design</h3> Instead of designing a new signaling solution from scratch, we chose to use the AppRTC application we already had. This has the additional benefit of testing the AppRTC code, which we are also maintaining. We could also have used the small <a href="https://code.google.com/p/chromium/codesearch#chromium/src/third_party/libjingle/libjingle.gyp&q=peerconnection_server&sq=package:chromium&type=cs&l=289">peerconnection_server</a> binary and some JavaScript, which would give us additional flexibility in what to test. We chose to go with AppRTC since it effectively implements the signaling for us, leading to much less test code. We assumed we would be able to get hold of the latest nightly Firefox and be able to launch that with a given URL. For the Chrome side, we assumed we would be running in a browser test, i.e. on a complete Chrome with some test scaffolding around it. For the first sketch of the test, we imagined just connecting the browsers to the live <a href="http://apprtc.appspot.com/">apprtc.appspot.com</a> with some random room number. If the call got established, we would be able to look at the remote video feed on the Chrome side and verify that video was playing (for instance using the <a href="http://googlechrome.github.io/webrtc/samples/web/content/getusermedia-canvas">video+canvas grab trick</a>). Furthermore, we could verify that audio was playing, for instance by using <a href="http://googlechrome.github.io/webrtc/samples/web/content/getusermedia-volume">WebRTC getStats</a> to measure the audio track energy level. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjGVzL0DcoTmuyYSQdoaMzOTbBqr2Si69Hv8NoDwRc6rHYQ4_PTNgmYcRHcoe78owHgH50kkA6wD7Ec3E0GsUARSbEvVx34zTXOyQVObb4DzlD41hRh80OzAuNXhXZYTD-Iy10F/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjGVzL0DcoTmuyYSQdoaMzOTbBqr2Si69Hv8NoDwRc6rHYQ4_PTNgmYcRHcoe78owHgH50kkA6wD7Ec3E0GsUARSbEvVx34zTXOyQVObb4DzlD41hRh80OzAuNXhXZYTD-Iy10F/s1600/image01.png" width="500" /></a></div> <div style="text-align: center;"> Figure 2. Basic test design.</div> However, since we like tests to be <a href="http://googletesting.blogspot.se/2012/10/hermetic-servers.html">hermetic</a>, this isn’t a good design. I can see several problems. For example, if the network between us and AppRTC is unreliable. Also, what if someone has occupied myroomid? If that were the case, the test would fail and we would be none the wiser. So to make this thing work, we would have to find some way to bring up the AppRTC instance on localhost to make our test hermetic. <h3> Bringing up AppRTC on localhost</h3> AppRTC is a <a href="https://developers.google.com/appengine/">Google App Engine application</a>. As this <a href="https://developers.google.com/appengine/docs/python/gettingstartedpython27/helloworld">hello world example</a> demonstrates, one can test applications locally with <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">google_appengine/dev_appserver.py apprtc_code/</pre> So why not just call this from our test? It turns out we need to solve some complicated problems first, like how to ensure the AppEngine SDK and the AppRTC code is actually available on the executing machine, but we’ll get to that later. Let’s assume for now that stuff is just available. We can now write the browser test code to launch the local instance: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool LaunchApprtcInstanceOnLocalhost() // ... Figure out locations of SDK and apprtc code ... CommandLine command_line(CommandLine::NO_PROGRAM); EXPECT_TRUE(GetPythonCommand(&command_line)); command_line.AppendArgPath(appengine_dev_appserver); command_line.AppendArgPath(apprtc_dir); command_line.AppendArg("--port=9999"); command_line.AppendArg("--admin_port=9998"); command_line.AppendArg("--skip_sdk_update_check"); VLOG(1) << "Running " << command_line.GetCommandLineString(); return base::LaunchProcess(command_line, base::LaunchOptions(), &dev_appserver_); } </pre> That’s pretty straightforward <a href="#foot1">[1]</a>. <h3> Figuring out Whether the Local Server is Up </h3> Then we ran into a very typical test problem. So we have the code to get the server up, and launching the two browsers to connect to http://localhost:9999?r=some_room is easy. But how do we know when to connect? When I first ran the test, it would work sometimes and sometimes not depending on if the server had time to get up. It’s tempting in these situations to just add a sleep to give the server time to get up. Don’t do that. That will result in a test that is flaky and/or slow. In these situations we need to identify what we’re really waiting for. We could probably monitor the stdout of the dev_appserver.py and look for some message that says “Server is up!” or equivalent. However, we’re really waiting for the server to be able to serve web pages, and since we have two browsers that are really good at connecting to servers, why not use them? Consider this code. <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">bool LocalApprtcInstanceIsUp() { // Load the admin page and see if we manage to load it right. ui_test_utils::NavigateToURL(browser(), GURL("localhost:9998")); content::WebContents* tab_contents = browser()->tab_strip_model()->GetActiveWebContents(); std::string javascript = "window.domAutomationController.send(document.title)"; std::string result; if (!content::ExecuteScriptAndExtractString(tab_contents, javascript, &result)) return false; return result == kTitlePageOfAppEngineAdminPage; } </pre> Here we ask Chrome to load the AppEngine admin page for the local server (we set the admin port to 9998 earlier, remember?) and ask it what its title is. If that title is “Instances”, the admin page has been displayed, and the server must be up. If the server isn’t up, Chrome will fail to load the page and the title will be something like “localhost:9999 is not available”. Then, we can just do this from the test: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">while (!LocalApprtcInstanceIsUp()) VLOG(1) << "Waiting for AppRTC to come up..."; </pre> If the server never comes up, for whatever reason, the test will just time out in that loop. If it comes up we can safely proceed with the rest of test. <h3> Launching the Browsers </h3> A browser window launches itself as a part of every Chromium browser test. It’s also easy for the test to control the command line switches the browser will run under. We have less control over the Firefox browser since it is the “foreign” browser in this test, but we can still pass command-line options to it when we invoke the Firefox process. To make this easier, Mozilla provides a Python library called mozrunner. Using that we can set up a <a href="http://chromegw/viewvc/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS/run_firefox_webrtc.py?revision=240895&view=markup">launcher python script</a> we can invoke from the test: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">from mozprofile import profile from mozrunner import runner WEBRTC_PREFERENCES = { 'media.navigator.permission.disabled': True, } def main(): # Set up flags, handle SIGTERM, etc # ... firefox_profile = profile.FirefoxProfile(preferences=WEBRTC_PREFERENCES) firefox_runner = runner.FirefoxRunner( profile=firefox_profile, binary=options.binary, cmdargs=[options.webpage]) firefox_runner.start()</pre> Notice that we need to pass special preferences to make Firefox accept the getUserMedia prompt. Otherwise, the test would get stuck on the prompt and we would be unable to set up a call. Alternatively, we could employ some kind of clickbot to click “Allow” on the prompt when it pops up, but that is way harder to set up. Without going into too much detail, the code for launching the browsers becomes <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">GURL room_url = GURL(base::StringPrintf("http://localhost:9999?r=room_%d", base::RandInt(0, 65536))); content::WebContents* chrome_tab = OpenPageAndAcceptUserMedia(room_url); ASSERT_TRUE(LaunchFirefoxWithUrl(room_url));</pre> Where LaunchFirefoxWithUrl essentially runs this: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">run_firefox_webrtc.py --binary /path/to/firefox --webpage http://localhost::9999?r=my_room</pre> Now we can launch the two browsers. Next time we will look at how we actually verify that the call worked, and how we actually download all resources needed by the test in a maintainable and automated manner. Stay tuned! <hr /> 1 The explicit ports are because the default ports collided on the bots we were running on, and the --skip_sdk_update_check was because the SDK stopped and asked us something if there was an update. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Chrome - Firefox WebRTC Interop Test - Pt 1&url=https://testing.googleblog.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund' rel='tag'> Patrik Höglund </a> </div> </div> </div> <div class='post' data-id='903802091935906990' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/08/testing-on-toilet-web-testing-made.html' itemprop='url' title='Testing on the Toilet: Web Testing Made Easier: Debug IDs'> Testing on the Toilet: Web Testing Made Easier: Debug IDs </a> </h2> <div class='post-header'> <div class='published'> Tuesday, August 12, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Ruslan Khamitov  This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet (TotT) episode</a>. You can download a <a href="https://docs.google.com/document/d/1zHky363AF_eNVGOgnpD-7ouhjTG7QbYmCmGwbeFKruE/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Adding ID attributes to elements can make it much easier to write tests that interact with the DOM (e.g., WebDriver tests). Consider the following DOM with two buttons that differ only by inner text: <table style="border-collapse: collapse; border: 1px solid black; width: 100%;"> <tbody> <tr> <td style="border: 1px solid black; text-align: center;">Save button</td> <td style="border: 1px solid black; text-align: center;">Edit button</td> </tr> <tr> <td style="background: #cfe2f3; border: 1px solid black;"><pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 0px; color: black; overflow: auto; padding: 0px 5px;"><div class="button">Save</div></pre> </td><td style="background: #cfe2f3; border: 1px solid black;"><pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 0px; color: black; overflow: auto; padding: 0px 5px;"><div class="button">Edit</div></pre> </td> </tr> </tbody></table> How would you tell WebDriver to interact with the “Save” button in this case? You have several options. One option is to interact with the button using a CSS selector: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">div.button</pre> However, this approach is not sufficient to identify a particular button, and there is no mechanism to filter by text in CSS. Another option would be to write an XPath, which is generally fragile and discouraged: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">//div[@class='button' and text()='Save']</pre> Your best option is to add unique hierarchical IDs where each widget is passed a base ID that it prepends to the ID of each of its children. The IDs for each button will be: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">contact-form.save-button contact-form.edit-button</pre> In GWT you can accomplish this by overriding onEnsureDebugId()on your widgets. Doing so allows you to create custom logic for applying debug IDs to the sub-elements that make up a custom widget: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Override protected void onEnsureDebugId(String baseId) { super.onEnsureDebugId(baseId); saveButton.ensureDebugId(baseId + ".save-button"); editButton.ensureDebugId(baseId + ".edit-button"); }</pre> Consider another example. Let’s set IDs for repeated UI elements in Angular using ng-repeat. Setting an index can help differentiate between repeated instances of each element: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><tr id="feedback-{{$index}}" class="feedback" ng-repeat="feedback in ctrl.feedbacks" ></pre> In GWT you can do this with ensureDebugId(). Let’s set an ID for each of the table cells: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@UiField FlexTable table; UIObject.ensureDebugId(table.getCellFormatter().getElement(rowIndex, columnIndex), baseID + colIndex + "-" + rowIndex);</pre> Take-away: Debug IDs are easy to set and make a huge difference for testing. Please add them early. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Ruslan Khamitov  This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet (TotT) episode</a>. You can download a <a href="https://docs.google.com/document/d/1zHky363AF_eNVGOgnpD-7ouhjTG7QbYmCmGwbeFKruE/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Adding ID attributes to elements can make it much easier to write tests that interact with the DOM (e.g., WebDriver tests). Consider the following DOM with two buttons that differ only by inner text: <table style="border-collapse: collapse; border: 1px solid black; width: 100%;"> <tbody> <tr> <td style="border: 1px solid black; text-align: center;">Save button</td> <td style="border: 1px solid black; text-align: center;">Edit button</td> </tr> <tr> <td style="background: #cfe2f3; border: 1px solid black;"><pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 0px; color: black; overflow: auto; padding: 0px 5px;"><div class="button">Save</div></pre> </td><td style="background: #cfe2f3; border: 1px solid black;"><pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 0px; color: black; overflow: auto; padding: 0px 5px;"><div class="button">Edit</div></pre> </td> </tr> </tbody></table> How would you tell WebDriver to interact with the “Save” button in this case? You have several options. One option is to interact with the button using a CSS selector: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">div.button</pre> However, this approach is not sufficient to identify a particular button, and there is no mechanism to filter by text in CSS. Another option would be to write an XPath, which is generally fragile and discouraged: <pre style="background: #cfe2f3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">//div[@class='button' and text()='Save']</pre> Your best option is to add unique hierarchical IDs where each widget is passed a base ID that it prepends to the ID of each of its children. The IDs for each button will be: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">contact-form.save-button contact-form.edit-button</pre> In GWT you can accomplish this by overriding onEnsureDebugId()on your widgets. Doing so allows you to create custom logic for applying debug IDs to the sub-elements that make up a custom widget: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Override protected void onEnsureDebugId(String baseId) { super.onEnsureDebugId(baseId); saveButton.ensureDebugId(baseId + ".save-button"); editButton.ensureDebugId(baseId + ".edit-button"); }</pre> Consider another example. Let’s set IDs for repeated UI elements in Angular using ng-repeat. Setting an index can help differentiate between repeated instances of each element: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"><tr id="feedback-{{$index}}" class="feedback" ng-repeat="feedback in ctrl.feedbacks" ></pre> In GWT you can do this with ensureDebugId(). Let’s set an ID for each of the table cells: <pre style="background: #d9ead3; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@UiField FlexTable table; UIObject.ensureDebugId(table.getCellFormatter().getElement(rowIndex, columnIndex), baseID + colIndex + "-" + rowIndex);</pre> Take-away: Debug IDs are easy to set and make a huge difference for testing. Please add them early. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Web Testing Made Easier: Debug IDs&url=https://testing.googleblog.com/2014/08/testing-on-toilet-web-testing-made.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/08/testing-on-toilet-web-testing-made.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/08/testing-on-toilet-web-testing-made.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov' rel='tag'> Ruslan Khamitov </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='3420885653031257617' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/07/testing-on-toilet-dont-put-logic-in.html' itemprop='url' title='Testing on the Toilet: Don't Put Logic in Tests'> Testing on the Toilet: Don't Put Logic in Tests </a> </h2> <div class='post-header'> <div class='published'> Thursday, July 31, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Erik Kuefler This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1f-ISrr6ItyDCJCMaFXOyQBZmekBZwxQ0tZeksoWUZAo/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Programming languages give us a lot of expressive power. Concepts like operators and conditionals are important tools that allow us to write programs that handle a wide range of inputs. But this flexibility comes at the cost of increased complexity, which makes our programs harder to understand. Unlike production code, simplicity is more important than flexibility in tests. Most unit tests verify that a single, known input produces a single, known output. Tests can avoid complexity by stating their inputs and outputs directly rather than computing them. Otherwise it's easy for tests to develop their own bugs. Let's take a look at a simple example. Does this test look correct to you? <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldNavigateToPhotosPage() { String baseUrl = "http://plus.google.com/"; Navigator nav = new Navigator(baseUrl); nav.goToPhotosPage(); assertEquals(baseUrl + "/u/0/photos", nav.getCurrentUrl()); }</pre> The author is trying to avoid duplication by storing a shared prefix in a variable. Performing a single string concatenation doesn't seem too bad, but what happens if we simplify the test by inlining the variable? <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldNavigateToPhotosPage() { Navigator nav = new Navigator("http://plus.google.com/"); nav.goToPhotosPage(); assertEquals("http://plus.google.com//u/0/photos", nav.getCurrentUrl()); // Oops! }</pre> After eliminating the unnecessary computation from the test, the bug is obvious—we're expecting two slashes in the URL! This test will either fail or (even worse) incorrectly pass if the production code has the same bug. We never would have written this if we stated our inputs and outputs directly instead of trying to compute them. And this is a very simple example—when a test adds more operators or includes loops and conditionals, it becomes increasingly difficult to be confident that it is correct. Another way of saying this is that, whereas production code describes a general strategy for computing outputs given inputs, tests are concrete examples of input/output pairs (where output might include side effects like verifying interactions with other classes). It's usually easy to tell whether an input/output pair is correct or not, even if the logic required to compute it is very complex. For instance, it's hard to picture the exact DOM that would be created by a Javascript function for a given server response. So the ideal test for such a function would just compare against a string containing the expected output HTML. When tests do need their own logic, such logic should often be moved out of the test bodies and into utilities and helper functions. Since such helpers can get quite complex, it's usually a good idea for any nontrivial test utility to have its own tests. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Erik Kuefler This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1f-ISrr6ItyDCJCMaFXOyQBZmekBZwxQ0tZeksoWUZAo/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Programming languages give us a lot of expressive power. Concepts like operators and conditionals are important tools that allow us to write programs that handle a wide range of inputs. But this flexibility comes at the cost of increased complexity, which makes our programs harder to understand. Unlike production code, simplicity is more important than flexibility in tests. Most unit tests verify that a single, known input produces a single, known output. Tests can avoid complexity by stating their inputs and outputs directly rather than computing them. Otherwise it's easy for tests to develop their own bugs. Let's take a look at a simple example. Does this test look correct to you? <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldNavigateToPhotosPage() { String baseUrl = "http://plus.google.com/"; Navigator nav = new Navigator(baseUrl); nav.goToPhotosPage(); assertEquals(baseUrl + "/u/0/photos", nav.getCurrentUrl()); }</pre> The author is trying to avoid duplication by storing a shared prefix in a variable. Performing a single string concatenation doesn't seem too bad, but what happens if we simplify the test by inlining the variable? <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldNavigateToPhotosPage() { Navigator nav = new Navigator("http://plus.google.com/"); nav.goToPhotosPage(); assertEquals("http://plus.google.com//u/0/photos", nav.getCurrentUrl()); // Oops! }</pre> After eliminating the unnecessary computation from the test, the bug is obvious—we're expecting two slashes in the URL! This test will either fail or (even worse) incorrectly pass if the production code has the same bug. We never would have written this if we stated our inputs and outputs directly instead of trying to compute them. And this is a very simple example—when a test adds more operators or includes loops and conditionals, it becomes increasingly difficult to be confident that it is correct. Another way of saying this is that, whereas production code describes a general strategy for computing outputs given inputs, tests are concrete examples of input/output pairs (where output might include side effects like verifying interactions with other classes). It's usually easy to tell whether an input/output pair is correct or not, even if the logic required to compute it is very complex. For instance, it's hard to picture the exact DOM that would be created by a Javascript function for a given server response. So the ideal test for such a function would just compare against a string containing the expected output HTML. When tests do need their own logic, such logic should often be moved out of the test bodies and into utilities and helper functions. Since such helpers can get quite complex, it's usually a good idea for any nontrivial test utility to have its own tests. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Don't Put Logic in Tests&url=https://testing.googleblog.com/2014/07/testing-on-toilet-dont-put-logic-in.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/07/testing-on-toilet-dont-put-logic-in.html#comments' style='font-weight: 500; text-decoration: underline;'>14 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/07/testing-on-toilet-dont-put-logic-in.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Erik%20Kuefler' rel='tag'> Erik Kuefler </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='8052289074541264722' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/07/the-deadline-to-sign-up-for-gtac-2014.html' itemprop='url' title='The Deadline to Sign up for GTAC 2014 is Jul 28'> The Deadline to Sign up for GTAC 2014 is Jul 28 </a> </h2> <div class='post-header'> <div class='published'> Tuesday, July 22, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee The deadline to sign up for GTAC 2014 is next Monday, July 28th, 2014. There is a great deal of interest to both attend and speak, and we’ve received many outstanding proposals. However, it’s not too late to add yours for consideration. If you would like to speak or attend, be sure to <a href="https://docs.google.com/a/google.com/forms/d/1HVm6KcFBdQAbhX_uh6LEjVYI1ZCCr-L9t7ocbtvLIMU/viewform">complete the form</a> by Monday. We will be making regular updates to our site over the next several weeks, and you can find conference details there:   <a href="http://developers.google.com/gtac">developers.google.com/gtac</a> For those that have already signed up to attend or speak, we will contact you directly in mid August. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee The deadline to sign up for GTAC 2014 is next Monday, July 28th, 2014. There is a great deal of interest to both attend and speak, and we’ve received many outstanding proposals. However, it’s not too late to add yours for consideration. If you would like to speak or attend, be sure to <a href="https://docs.google.com/a/google.com/forms/d/1HVm6KcFBdQAbhX_uh6LEjVYI1ZCCr-L9t7ocbtvLIMU/viewform">complete the form</a> by Monday. We will be making regular updates to our site over the next several weeks, and you can find conference details there:   <a href="http://developers.google.com/gtac">developers.google.com/gtac</a> For those that have already signed up to attend or speak, we will contact you directly in mid August. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:The Deadline to Sign up for GTAC 2014 is Jul 28&url=https://testing.googleblog.com/2014/07/the-deadline-to-sign-up-for-gtac-2014.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/07/the-deadline-to-sign-up-for-gtac-2014.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/07/the-deadline-to-sign-up-for-gtac-2014.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='9102542563573191505' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/07/measuring-coverage-at-google.html' itemprop='url' title='Measuring Coverage at Google'> Measuring Coverage at Google </a> </h2> <div class='post-header'> <div class='published'> Monday, July 14, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Marko Ivanković, Google Zürich <h3>Introduction</h3> <a href="http://en.wikipedia.org/wiki/Code_coverage">Code coverage</a> is a very interesting metric, covered by a <a href="http://scholar.google.com/scholar?q=%22code+coverage%22">large body of research</a> that reaches somewhat contradictory results. Some people think it is an extremely useful metric and that a certain percentage of coverage should be enforced on all code. Some think it is a useful tool to identify areas that need more testing but don’t necessarily trust that covered code is truly well tested. Others yet think that measuring coverage is actively harmful because it provides a false sense of security. Our team’s mission was to collect coverage related data then develop and champion code coverage practices across Google. We designed an opt-in system where engineers could enable two different types of coverage measurements for their projects: daily and per-commit. With daily coverage, we run all tests for their project, where as with per-commit coverage we run only the tests affected by the commit. The two measurements are independent and many projects opted into both. While we did experiment with branch, function and statement coverage, we ended up focusing mostly on statement coverage because of its relative simplicity and ease of visualization. <h3> How we measured </h3> Our job was made significantly easier by the wonderful <a href="http://google-engtools.blogspot.ch/2011/08/build-in-cloud-how-build-system-works.html">Google build system</a> whose parallelism and flexibility allowed us to simply scale our measurements to Google scale. The build system had integrated various language-specific open source coverage measurement tools like <a href="https://gcc.gnu.org/onlinedocs/gcc/Gcov.html">Gcov</a> (C++), <a href="http://emma.sourceforge.net/">Emma</a> / <a href="http://www.eclemma.org/jacoco/">JaCoCo</a> (Java) and <a href="https://pypi.python.org/pypi/coverage/3.7.1">Coverage.py</a> (Python), and we provided a central system where teams could sign up for coverage measurement. For daily whole project coverage measurements, each team was provided with a simple cronjob that would run all tests across the project’s codebase. The results of these runs were available to the teams in a centralized dashboard that displays charts showing coverage over time and allows daily / weekly / quarterly / yearly aggregations and per-language slicing. On this dashboard teams can also compare their project (or projects) with any other project, or Google as a whole. For per-commit measurement, we hook into the Google code review process (<a href="http://googletesting.blogspot.ch/2014/01/the-google-test-and-development_21.html">briefly explained in this article</a>) and display the data visually to both the commit author and the reviewers. We display the data on two levels: color coded lines right next to the color coded diff and a total aggregate number for the entire commit. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh47CDtSWJ7AfsJ5J9fq9_PtHIaoNAHbEYFu5xnNgBE11fQjp6TgG3uaqFWjcwxhOXbXYc5sAKsEIipOseYx62Kr4uvxPzTdWYT5IQvI0zc7s12fgbumextd9Po4V6HZCdfuHPo/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="145" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh47CDtSWJ7AfsJ5J9fq9_PtHIaoNAHbEYFu5xnNgBE11fQjp6TgG3uaqFWjcwxhOXbXYc5sAKsEIipOseYx62Kr4uvxPzTdWYT5IQvI0zc7s12fgbumextd9Po4V6HZCdfuHPo/s1600/image01.png" width="400" /></a></div> Displayed above is a screenshot of the code review tool. The green line coloring is the standard diff coloring for added lines. The orange and lighter green coloring on the line numbers is the coverage information. We use light green for covered lines, orange for non-covered lines and white for non-instrumented lines. It’s important to note that we surface the coverage information before the commit is submitted to the codebase, because this is the time when engineers are most likely to be interested in improving it. <h3> Results</h3> One of the main benefits of working at Google is the scale at which we operate. We have been running the coverage measurement system for some time now and we have collected data for more than 650 different projects, spanning 100,000+ commits. We have a significant amount of data for C++, Java, Python, Go and JavaScript code. I am happy to say that we can share some preliminary results with you today: <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjHD0LGE5yH176DcZSLpS6bpA5RpAA9MQFWglQoSYHVV4I4n_4v8q39xRseDvno3o_f8KnBL43lEKnk-bJz9smd2XYC9UEBjSFZV5Jhzid0rhNQq3bF4b6Utu8IrKQL8NeT9fc4/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="262" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjHD0LGE5yH176DcZSLpS6bpA5RpAA9MQFWglQoSYHVV4I4n_4v8q39xRseDvno3o_f8KnBL43lEKnk-bJz9smd2XYC9UEBjSFZV5Jhzid0rhNQq3bF4b6Utu8IrKQL8NeT9fc4/s1600/image00.png" width="640" /></a></div> The chart above is the histogram of average values of measured absolute coverage across Google. The median (50th percentile) code coverage is 78%, the 75th percentile 85% and 90th percentile 90%. We believe that these numbers represent a very healthy codebase. We have also found it very interesting that there are significant differences between languages: <table style="border-collapse: collapse; border: 1px solid black; width: 100%;"> <tbody> <tr> <td style="border: 1px solid black; padding: 5px;">C++</td> <td style="border: 1px solid black; padding: 5px;">Java</td> <td style="border: 1px solid black; padding: 5px;">Go</td> <td style="border: 1px solid black; padding: 5px;">JavaScript</td> <td style="border: 1px solid black; padding: 5px;">Python</td> </tr> <tr> <td style="border: 1px solid black; padding: 5px;">56.6%</td> <td style="border: 1px solid black; padding: 5px;">61.2%</td> <td style="border: 1px solid black; padding: 5px;">63.0%</td> <td style="border: 1px solid black; padding: 5px;">76.9%</td> <td style="border: 1px solid black; padding: 5px;">84.2%</td> </tr> </tbody></table> The table above shows the total coverage of all analyzed code for each language, averaged over the past quarter. We believe that the large difference is due to structural, paradigm and best practice differences between languages and the more precise ability to measure coverage in certain languages. Note that these numbers should not be interpreted as guidelines for a particular language, the aggregation method used is too simple for that. Instead this finding is simply a data point for any future research that analyzes samples from a single programming language. The feedback from our fellow engineers was overwhelmingly positive. The most loved feature was surfacing the coverage information during code review time. This early surfacing of coverage had a statistically significant impact: our initial analysis suggests that it increased coverage by 10% (averaged across all commits). <h3> Future work</h3> We are aware that there are a few problems with the dataset we collected. In particular, the individual tools we use to measure coverage are not perfect. Large integration tests, end to end tests and UI tests are difficult to instrument, so large parts of code exercised by such tests can be misreported as non-covered. We are working on improving the tools, but also analyzing the impact of unit tests, integration tests and other types of tests individually. In addition to languages, we will also investigate other factors that might influence coverage, such as platforms and frameworks, to allow all future research to account for their effect. We will be publishing more of our findings in the future, so stay tuned. And if this sounds like something you would like to work on, why not apply on our <a href="//goo.gl/tFRejX">job site</a>? <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Marko Ivanković, Google Zürich <h3>Introduction</h3> <a href="http://en.wikipedia.org/wiki/Code_coverage">Code coverage</a> is a very interesting metric, covered by a <a href="http://scholar.google.com/scholar?q=%22code+coverage%22">large body of research</a> that reaches somewhat contradictory results. Some people think it is an extremely useful metric and that a certain percentage of coverage should be enforced on all code. Some think it is a useful tool to identify areas that need more testing but don’t necessarily trust that covered code is truly well tested. Others yet think that measuring coverage is actively harmful because it provides a false sense of security. Our team’s mission was to collect coverage related data then develop and champion code coverage practices across Google. We designed an opt-in system where engineers could enable two different types of coverage measurements for their projects: daily and per-commit. With daily coverage, we run all tests for their project, where as with per-commit coverage we run only the tests affected by the commit. The two measurements are independent and many projects opted into both. While we did experiment with branch, function and statement coverage, we ended up focusing mostly on statement coverage because of its relative simplicity and ease of visualization. <h3> How we measured </h3> Our job was made significantly easier by the wonderful <a href="http://google-engtools.blogspot.ch/2011/08/build-in-cloud-how-build-system-works.html">Google build system</a> whose parallelism and flexibility allowed us to simply scale our measurements to Google scale. The build system had integrated various language-specific open source coverage measurement tools like <a href="https://gcc.gnu.org/onlinedocs/gcc/Gcov.html">Gcov</a> (C++), <a href="http://emma.sourceforge.net/">Emma</a> / <a href="http://www.eclemma.org/jacoco/">JaCoCo</a> (Java) and <a href="https://pypi.python.org/pypi/coverage/3.7.1">Coverage.py</a> (Python), and we provided a central system where teams could sign up for coverage measurement. For daily whole project coverage measurements, each team was provided with a simple cronjob that would run all tests across the project’s codebase. The results of these runs were available to the teams in a centralized dashboard that displays charts showing coverage over time and allows daily / weekly / quarterly / yearly aggregations and per-language slicing. On this dashboard teams can also compare their project (or projects) with any other project, or Google as a whole. For per-commit measurement, we hook into the Google code review process (<a href="http://googletesting.blogspot.ch/2014/01/the-google-test-and-development_21.html">briefly explained in this article</a>) and display the data visually to both the commit author and the reviewers. We display the data on two levels: color coded lines right next to the color coded diff and a total aggregate number for the entire commit. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh47CDtSWJ7AfsJ5J9fq9_PtHIaoNAHbEYFu5xnNgBE11fQjp6TgG3uaqFWjcwxhOXbXYc5sAKsEIipOseYx62Kr4uvxPzTdWYT5IQvI0zc7s12fgbumextd9Po4V6HZCdfuHPo/s1600/image01.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="145" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh47CDtSWJ7AfsJ5J9fq9_PtHIaoNAHbEYFu5xnNgBE11fQjp6TgG3uaqFWjcwxhOXbXYc5sAKsEIipOseYx62Kr4uvxPzTdWYT5IQvI0zc7s12fgbumextd9Po4V6HZCdfuHPo/s1600/image01.png" width="400" /></a></div> Displayed above is a screenshot of the code review tool. The green line coloring is the standard diff coloring for added lines. The orange and lighter green coloring on the line numbers is the coverage information. We use light green for covered lines, orange for non-covered lines and white for non-instrumented lines. It’s important to note that we surface the coverage information before the commit is submitted to the codebase, because this is the time when engineers are most likely to be interested in improving it. <h3> Results</h3> One of the main benefits of working at Google is the scale at which we operate. We have been running the coverage measurement system for some time now and we have collected data for more than 650 different projects, spanning 100,000+ commits. We have a significant amount of data for C++, Java, Python, Go and JavaScript code. I am happy to say that we can share some preliminary results with you today: <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjHD0LGE5yH176DcZSLpS6bpA5RpAA9MQFWglQoSYHVV4I4n_4v8q39xRseDvno3o_f8KnBL43lEKnk-bJz9smd2XYC9UEBjSFZV5Jhzid0rhNQq3bF4b6Utu8IrKQL8NeT9fc4/s1600/image00.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="262" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjHD0LGE5yH176DcZSLpS6bpA5RpAA9MQFWglQoSYHVV4I4n_4v8q39xRseDvno3o_f8KnBL43lEKnk-bJz9smd2XYC9UEBjSFZV5Jhzid0rhNQq3bF4b6Utu8IrKQL8NeT9fc4/s1600/image00.png" width="640" /></a></div> The chart above is the histogram of average values of measured absolute coverage across Google. The median (50th percentile) code coverage is 78%, the 75th percentile 85% and 90th percentile 90%. We believe that these numbers represent a very healthy codebase. We have also found it very interesting that there are significant differences between languages: <table style="border-collapse: collapse; border: 1px solid black; width: 100%;"> <tbody> <tr> <td style="border: 1px solid black; padding: 5px;">C++</td> <td style="border: 1px solid black; padding: 5px;">Java</td> <td style="border: 1px solid black; padding: 5px;">Go</td> <td style="border: 1px solid black; padding: 5px;">JavaScript</td> <td style="border: 1px solid black; padding: 5px;">Python</td> </tr> <tr> <td style="border: 1px solid black; padding: 5px;">56.6%</td> <td style="border: 1px solid black; padding: 5px;">61.2%</td> <td style="border: 1px solid black; padding: 5px;">63.0%</td> <td style="border: 1px solid black; padding: 5px;">76.9%</td> <td style="border: 1px solid black; padding: 5px;">84.2%</td> </tr> </tbody></table> The table above shows the total coverage of all analyzed code for each language, averaged over the past quarter. We believe that the large difference is due to structural, paradigm and best practice differences between languages and the more precise ability to measure coverage in certain languages. Note that these numbers should not be interpreted as guidelines for a particular language, the aggregation method used is too simple for that. Instead this finding is simply a data point for any future research that analyzes samples from a single programming language. The feedback from our fellow engineers was overwhelmingly positive. The most loved feature was surfacing the coverage information during code review time. This early surfacing of coverage had a statistically significant impact: our initial analysis suggests that it increased coverage by 10% (averaged across all commits). <h3> Future work</h3> We are aware that there are a few problems with the dataset we collected. In particular, the individual tools we use to measure coverage are not perfect. Large integration tests, end to end tests and UI tests are difficult to instrument, so large parts of code exercised by such tests can be misreported as non-covered. We are working on improving the tools, but also analyzing the impact of unit tests, integration tests and other types of tests individually. In addition to languages, we will also investigate other factors that might influence coverage, such as platforms and frameworks, to allow all future research to account for their effect. We will be publishing more of our findings in the future, so stay tuned. And if this sounds like something you would like to work on, why not apply on our <a href="//goo.gl/tFRejX">job site</a>? <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Measuring Coverage at Google&url=https://testing.googleblog.com/2014/07/measuring-coverage-at-google.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/07/measuring-coverage-at-google.html#comments' style='font-weight: 500; text-decoration: underline;'>15 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/07/measuring-coverage-at-google.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87' rel='tag'> Marko Ivanković </a> </div> </div> </div> <div class='post' data-id='431940925295179551' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/06/threadsanitizer-slaughtering-data-races.html' itemprop='url' title='ThreadSanitizer: Slaughtering Data Races'> ThreadSanitizer: Slaughtering Data Races </a> </h2> <div class='post-header'> <div class='published'> Monday, June 30, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Dmitry Vyukov, Synchronization Lookout, Google, Moscow Hello, I work in the Dynamic Testing Tools team at Google. Our team develops tools like <a href="http://address-sanitizer.googlecode.com/">AddressSanitizer</a>, <a href="http://memory-sanitizer.googlecode.com/">MemorySanitizer</a> and <a href="http://thread-sanitizer.googlecode.com/">ThreadSanitizer</a> which find various kinds of bugs. In this blog post I want to tell you about ThreadSanitizer, a fast data race detector for C++ and Go programs. First of all, what is a <a href="http://en.wikipedia.org/wiki/Race_condition">data race</a>? A data race occurs when two threads access the same variable concurrently, and at least one of the accesses attempts is a write. Most programming languages provide very weak guarantees, or no guarantees at all, for programs with data races. For example, in C++ absolutely any data race renders the behavior of the whole program as completely undefined (yes, it can suddenly format the hard drive). Data races are common in concurrent programs, and they are notoriously hard to debug and localize. A typical manifestation of a data race is when a program occasionally crashes with obscure symptoms, the symptoms are different each time and do not point to any particular place in the source code. Such bugs can take several months of debugging without particular success, since typical debugging techniques do not work. Fortunately, ThreadSanitizer can catch most data races in the blink of an eye. See Chromium <a href="https://code.google.com/p/chromium/issues/detail?id=15577">issue 15577</a> for an example of such a data race and <a href="https://code.google.com/p/chromium/issues/detail?id=18488">issue 18488</a> for the resolution. Due to the complex nature of bugs caught by ThreadSanitizer, we don't suggest waiting until product release validation to use the tool. For example, in Google, we've made our tools easily accessible to programmers during development, so that anyone can use the tool for testing if they suspect that new code might introduce a race. For both Chromium and Google internal server codebase, we run unit tests that use the tool continuously. This catches many regressions instantly. The Chromium project has recently started using ThreadSanitizer on <a href="http://blog.chromium.org/2012/04/fuzzing-for-security.html">ClusterFuzz</a>, a large scale fuzzing system. Finally, some teams also set up periodic end-to-end testing with ThreadSanitizer under a realistic workload, which proves to be extremely valuable. When races are found by the tool, our team has zero tolerance for races and does not consider any race to be benign, as even the most benign races can lead to <a href="https://software.intel.com/en-us/blogs/2013/01/06/benign-data-races-what-could-possibly-go-wrong">memory corruption</a>. Our tools are <a href="http://en.wikipedia.org/wiki/Dynamic_program_analysis">dynamic</a> (as opposed to <a href="http://en.wikipedia.org/wiki/Static_analysis_tool">static tools</a>). This means that they do not merely "look" at the code and try to surmise where bugs can be; instead they they <a href="http://en.wikipedia.org/wiki/Instrumentation_(computer_programming)">instrument</a> the binary at build time and then analyze dynamic behavior of the program to catch it red-handed. This approach has its pros and cons. On one hand, the tool does not have any false positives, thus it does not bother a developer with something that is not a bug. On the other hand, in order to catch a bug, the test must expose a bug -- the racing data access attempts must be executed in different threads. This requires writing good multi-threaded tests and makes end-to-end testing especially effective. As a bonus, ThreadSanitizer finds some other types of bugs: thread leaks, deadlocks, incorrect uses of mutexes, malloc calls in signal handlers, and <a href="https://code.google.com/p/thread-sanitizer/wiki/DetectableBugs">more</a>. It also natively understands atomic operations and thus can find bugs in <a href="http://en.wikipedia.org/wiki/Lock-free">lock-free</a> algorithms (see e.g. <a href="https://code.google.com/p/chromium/issues/detail?id=330528">this bug</a> in the V8 concurrent garbage collector). The tool is supported by both Clang and GCC compilers (only on Linux/Intel64). Using it is very simple: you just need to add a -fsanitize=thread flag during compilation and linking. For Go programs, you simply need to add a -race flag to the go tool (supported on Linux, Mac and Windows). Interestingly, after integrating the tool into compilers, we've found some bugs in the compilers themselves. For example, LLVM was <a href="http://llvm.org/bugs/show_bug.cgi?id=13691">illegally widening stores</a>, which can introduce very harmful data races into otherwise correct programs. And GCC was injecting <a href="https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48076">unsafe code</a> for initialization of function static variables. Among our other trophies are more than <a href="https://code.google.com/p/thread-sanitizer/wiki/FoundBugs">a thousand bugs</a> in Chromium, Firefox, the Go standard library, WebRTC, OpenSSL, and of course in our internal projects. So what are you waiting for? You know what to do! <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Dmitry Vyukov, Synchronization Lookout, Google, Moscow Hello, I work in the Dynamic Testing Tools team at Google. Our team develops tools like <a href="http://address-sanitizer.googlecode.com/">AddressSanitizer</a>, <a href="http://memory-sanitizer.googlecode.com/">MemorySanitizer</a> and <a href="http://thread-sanitizer.googlecode.com/">ThreadSanitizer</a> which find various kinds of bugs. In this blog post I want to tell you about ThreadSanitizer, a fast data race detector for C++ and Go programs. First of all, what is a <a href="http://en.wikipedia.org/wiki/Race_condition">data race</a>? A data race occurs when two threads access the same variable concurrently, and at least one of the accesses attempts is a write. Most programming languages provide very weak guarantees, or no guarantees at all, for programs with data races. For example, in C++ absolutely any data race renders the behavior of the whole program as completely undefined (yes, it can suddenly format the hard drive). Data races are common in concurrent programs, and they are notoriously hard to debug and localize. A typical manifestation of a data race is when a program occasionally crashes with obscure symptoms, the symptoms are different each time and do not point to any particular place in the source code. Such bugs can take several months of debugging without particular success, since typical debugging techniques do not work. Fortunately, ThreadSanitizer can catch most data races in the blink of an eye. See Chromium <a href="https://code.google.com/p/chromium/issues/detail?id=15577">issue 15577</a> for an example of such a data race and <a href="https://code.google.com/p/chromium/issues/detail?id=18488">issue 18488</a> for the resolution. Due to the complex nature of bugs caught by ThreadSanitizer, we don't suggest waiting until product release validation to use the tool. For example, in Google, we've made our tools easily accessible to programmers during development, so that anyone can use the tool for testing if they suspect that new code might introduce a race. For both Chromium and Google internal server codebase, we run unit tests that use the tool continuously. This catches many regressions instantly. The Chromium project has recently started using ThreadSanitizer on <a href="http://blog.chromium.org/2012/04/fuzzing-for-security.html">ClusterFuzz</a>, a large scale fuzzing system. Finally, some teams also set up periodic end-to-end testing with ThreadSanitizer under a realistic workload, which proves to be extremely valuable. When races are found by the tool, our team has zero tolerance for races and does not consider any race to be benign, as even the most benign races can lead to <a href="https://software.intel.com/en-us/blogs/2013/01/06/benign-data-races-what-could-possibly-go-wrong">memory corruption</a>. Our tools are <a href="http://en.wikipedia.org/wiki/Dynamic_program_analysis">dynamic</a> (as opposed to <a href="http://en.wikipedia.org/wiki/Static_analysis_tool">static tools</a>). This means that they do not merely "look" at the code and try to surmise where bugs can be; instead they they <a href="http://en.wikipedia.org/wiki/Instrumentation_(computer_programming)">instrument</a> the binary at build time and then analyze dynamic behavior of the program to catch it red-handed. This approach has its pros and cons. On one hand, the tool does not have any false positives, thus it does not bother a developer with something that is not a bug. On the other hand, in order to catch a bug, the test must expose a bug -- the racing data access attempts must be executed in different threads. This requires writing good multi-threaded tests and makes end-to-end testing especially effective. As a bonus, ThreadSanitizer finds some other types of bugs: thread leaks, deadlocks, incorrect uses of mutexes, malloc calls in signal handlers, and <a href="https://code.google.com/p/thread-sanitizer/wiki/DetectableBugs">more</a>. It also natively understands atomic operations and thus can find bugs in <a href="http://en.wikipedia.org/wiki/Lock-free">lock-free</a> algorithms (see e.g. <a href="https://code.google.com/p/chromium/issues/detail?id=330528">this bug</a> in the V8 concurrent garbage collector). The tool is supported by both Clang and GCC compilers (only on Linux/Intel64). Using it is very simple: you just need to add a -fsanitize=thread flag during compilation and linking. For Go programs, you simply need to add a -race flag to the go tool (supported on Linux, Mac and Windows). Interestingly, after integrating the tool into compilers, we've found some bugs in the compilers themselves. For example, LLVM was <a href="http://llvm.org/bugs/show_bug.cgi?id=13691">illegally widening stores</a>, which can introduce very harmful data races into otherwise correct programs. And GCC was injecting <a href="https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48076">unsafe code</a> for initialization of function static variables. Among our other trophies are more than <a href="https://code.google.com/p/thread-sanitizer/wiki/FoundBugs">a thousand bugs</a> in Chromium, Firefox, the Go standard library, WebRTC, OpenSSL, and of course in our internal projects. So what are you waiting for? You know what to do! <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:ThreadSanitizer: Slaughtering Data Races&url=https://testing.googleblog.com/2014/06/threadsanitizer-slaughtering-data-races.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/06/threadsanitizer-slaughtering-data-races.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/06/threadsanitizer-slaughtering-data-races.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov' rel='tag'> Dmitry Vyukov </a> </div> </div> </div> <div class='post' data-id='529430683170666561' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/06/gtac-2014-call-for-proposals-attendance.html' itemprop='url' title='GTAC 2014: Call for Proposals & Attendance'> GTAC 2014: Call for Proposals & Attendance </a> </h2> <div class='post-header'> <div class='published'> Monday, June 16, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee The application process is now open for presentation proposals and attendance for GTAC (Google Test Automation Conference) (<a href="http://googletesting.blogspot.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html">see initial announcement</a>) to be held at the <a href="//www.google.com/about/careers/locations/seattle-kirkland/">Google Kirkland office (near Seattle, WA)</a> on October 28 - 29th, 2014. GTAC will be streamed live on YouTube again this year, so even if you can’t attend, you’ll be able to watch the conference from your computer. Speakers Presentations are targeted at student, academic, and experienced engineers working on test automation. Full presentations and lightning talks are 45 minutes and 15 minutes respectively. Speakers should be prepared for a question and answer session following their presentation. Application For presentation proposals and/or attendance, <a href="https://docs.google.com/forms/d/1HVm6KcFBdQAbhX_uh6LEjVYI1ZCCr-L9t7ocbtvLIMU/viewform?usp=send_form">complete this form</a>. We will be selecting about 300 applicants for the event. Deadline The due date for both presentation and attendance applications is July 28, 2014. Fees There are no registration fees, and we will send out detailed registration instructions to each invited applicant. Meals will be provided, but speakers and attendees must arrange and pay for their own travel and accommodations. Update : Our <a href="https://developers.google.com/google-test-automation-conference/2014/contact">contact</a> email was bouncing - this is now fixed. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiJZm7T54cLri6bxgM440YLDoof1tJHTFvxs8cnHD8pdOs-x6tivTLYyymGXZoiVsJLjd_aU7dvREy0BPr_2jRairzWzhq1dZJrfpzi8i7ILEqiQBGuHwk13iti_8ranczWI4mf/s1600/GTAC+2014+logo.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="200" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiJZm7T54cLri6bxgM440YLDoof1tJHTFvxs8cnHD8pdOs-x6tivTLYyymGXZoiVsJLjd_aU7dvREy0BPr_2jRairzWzhq1dZJrfpzi8i7ILEqiQBGuHwk13iti_8ranczWI4mf/s1600/GTAC+2014+logo.png" width="200" /></a></div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee The application process is now open for presentation proposals and attendance for GTAC (Google Test Automation Conference) (<a href="http://googletesting.blogspot.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html">see initial announcement</a>) to be held at the <a href="//www.google.com/about/careers/locations/seattle-kirkland/">Google Kirkland office (near Seattle, WA)</a> on October 28 - 29th, 2014. GTAC will be streamed live on YouTube again this year, so even if you can’t attend, you’ll be able to watch the conference from your computer. Speakers Presentations are targeted at student, academic, and experienced engineers working on test automation. Full presentations and lightning talks are 45 minutes and 15 minutes respectively. Speakers should be prepared for a question and answer session following their presentation. Application For presentation proposals and/or attendance, <a href="https://docs.google.com/forms/d/1HVm6KcFBdQAbhX_uh6LEjVYI1ZCCr-L9t7ocbtvLIMU/viewform?usp=send_form">complete this form</a>. We will be selecting about 300 applicants for the event. Deadline The due date for both presentation and attendance applications is July 28, 2014. Fees There are no registration fees, and we will send out detailed registration instructions to each invited applicant. Meals will be provided, but speakers and attendees must arrange and pay for their own travel and accommodations. Update : Our <a href="https://developers.google.com/google-test-automation-conference/2014/contact">contact</a> email was bouncing - this is now fixed. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiJZm7T54cLri6bxgM440YLDoof1tJHTFvxs8cnHD8pdOs-x6tivTLYyymGXZoiVsJLjd_aU7dvREy0BPr_2jRairzWzhq1dZJrfpzi8i7ILEqiQBGuHwk13iti_8ranczWI4mf/s1600/GTAC+2014+logo.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="200" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiJZm7T54cLri6bxgM440YLDoof1tJHTFvxs8cnHD8pdOs-x6tivTLYyymGXZoiVsJLjd_aU7dvREy0BPr_2jRairzWzhq1dZJrfpzi8i7ILEqiQBGuHwk13iti_8ranczWI4mf/s1600/GTAC+2014+logo.png" width="200" /></a></div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2014: Call for Proposals & Attendance&url=https://testing.googleblog.com/2014/06/gtac-2014-call-for-proposals-attendance.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/06/gtac-2014-call-for-proposals-attendance.html#comments' style='font-weight: 500; text-decoration: underline;'>7 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/06/gtac-2014-call-for-proposals-attendance.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='620240275558562137' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html' itemprop='url' title='GTAC 2014 Coming to Seattle/Kirkland in October'> GTAC 2014 Coming to Seattle/Kirkland in October </a> </h2> <div class='post-header'> <div class='published'> Wednesday, June 04, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee If you're looking for a place to discuss the latest innovations in test automation, then charge your tablets and pack your <a href="http://en.wikipedia.org/wiki/Wellington_boot">gumboots</a> - the eighth <a href="https://developers.google.com/gtac/">GTAC</a> (Google Test Automation Conference) will be held on October 28-29, 2014 at Google Kirkland! The Kirkland office is part of the <a href="//www.google.com/about/careers/locations/seattle-kirkland/">Seattle/Kirkland campus</a> in beautiful Washington state. This campus forms our third largest engineering office in the USA. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiNsN48cVaYYCPqcM2tTFuNKwdgIOLJdFnfK7siAZZnLgFCESJY6OgM7XipRU8QgEua_nhPJOpebaHjWqPr3hOPRNb4UIrrwszxN1yoOHf7QzXKnIXw8S-Yv-T6zD4BRIj74wRJ/s1600/GTAC+2014+logo.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="200" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiNsN48cVaYYCPqcM2tTFuNKwdgIOLJdFnfK7siAZZnLgFCESJY6OgM7XipRU8QgEua_nhPJOpebaHjWqPr3hOPRNb4UIrrwszxN1yoOHf7QzXKnIXw8S-Yv-T6zD4BRIj74wRJ/s1600/GTAC+2014+logo.png" width="200" /></a></div> GTAC is a periodic conference hosted by Google, bringing together engineers from industry and academia to discuss advances in test automation and the test engineering computer science field. It’s a great opportunity to present, learn, and challenge modern testing technologies and strategies. You can browse the presentation abstracts, slides, and videos from last year on the <a href="https://developers.google.com/google-test-automation-conference/2013/">GTAC 2013 page</a>. Stay tuned to this blog and the GTAC website for application information and opportunities to present at GTAC. Subscribing to this blog is the best way to get notified. We're looking forward to seeing you there! <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> Posted by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee If you're looking for a place to discuss the latest innovations in test automation, then charge your tablets and pack your <a href="http://en.wikipedia.org/wiki/Wellington_boot">gumboots</a> - the eighth <a href="https://developers.google.com/gtac/">GTAC</a> (Google Test Automation Conference) will be held on October 28-29, 2014 at Google Kirkland! The Kirkland office is part of the <a href="//www.google.com/about/careers/locations/seattle-kirkland/">Seattle/Kirkland campus</a> in beautiful Washington state. This campus forms our third largest engineering office in the USA. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiNsN48cVaYYCPqcM2tTFuNKwdgIOLJdFnfK7siAZZnLgFCESJY6OgM7XipRU8QgEua_nhPJOpebaHjWqPr3hOPRNb4UIrrwszxN1yoOHf7QzXKnIXw8S-Yv-T6zD4BRIj74wRJ/s1600/GTAC+2014+logo.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="200" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiNsN48cVaYYCPqcM2tTFuNKwdgIOLJdFnfK7siAZZnLgFCESJY6OgM7XipRU8QgEua_nhPJOpebaHjWqPr3hOPRNb4UIrrwszxN1yoOHf7QzXKnIXw8S-Yv-T6zD4BRIj74wRJ/s1600/GTAC+2014+logo.png" width="200" /></a></div> GTAC is a periodic conference hosted by Google, bringing together engineers from industry and academia to discuss advances in test automation and the test engineering computer science field. It’s a great opportunity to present, learn, and challenge modern testing technologies and strategies. You can browse the presentation abstracts, slides, and videos from last year on the <a href="https://developers.google.com/google-test-automation-conference/2013/">GTAC 2013 page</a>. Stay tuned to this blog and the GTAC website for application information and opportunities to present at GTAC. Subscribing to this blog is the best way to get notified. We're looking forward to seeing you there! <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2014 Coming to Seattle/Kirkland in October&url=https://testing.googleblog.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='4863219221345478243' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/05/testing-on-toilet-risk-driven-testing.html' itemprop='url' title='Testing on the Toilet: Risk-Driven Testing'> Testing on the Toilet: Risk-Driven Testing </a> </h2> <div class='post-header'> <div class='published'> Friday, May 30, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Peter Arrenbrecht This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1QfmhsvpiCx__rcU0sr03Lu1fM1yhE8P9atyTgeLbRAQ/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. We are all conditioned to write tests as we code: unit, functional, UI—the whole shebang. We are professionals, after all. Many of us like how small tests let us work quickly, and how larger tests inspire safety and closure. Or we may just anticipate flak during review. We are so used to these tests that often we no longer question why we write them. This can be wasteful and dangerous. Tests are a means to an end: To reduce the key risks of a project, and to get the biggest bang for the buck. This bang may not always come from the tests that standard practice has you write, or not even from tests at all. Two examples: <div style="background: #fef2cc; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> “We built a new debugging aid. We wrote unit, integration, and UI tests. We were ready to launch.” Outstanding practice. Missing the mark. Our key risks were that we'd corrupt our data or bring down our servers for the sake of a debugging aid. None of the tests addressed this, but they gave a false sense of safety and “being done”. We stopped the launch.</div> <div style="background: #fef2cc; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> “We wanted to turn down a feature, so we needed to alert affected users. Again we had unit and integration tests, and even one expensive end-to-end test.” Standard practice. Wasted effort. The alert was so critical it actually needed end-to-end coverage for all scenarios. But it would be live for only three releases. The cheapest effective test? Manual testing before each release.</div> A Better Approach: Risks First For every project or feature, think about testing. Brainstorm your key risks and your best options to reduce them. Do this at the start so you don't waste effort and can adapt your design. Write them down as a QA design so you can point to it in reviews and discussions. To be sure, standard practice remains a good idea in most cases (hence it’s standard). Small tests are cheap and speed up coding and maintenance, and larger tests safeguard core use-cases and integration. Just remember: Your tests are a means. The bang is what counts. It’s your job to maximize it. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Peter Arrenbrecht This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1QfmhsvpiCx__rcU0sr03Lu1fM1yhE8P9atyTgeLbRAQ/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. We are all conditioned to write tests as we code: unit, functional, UI—the whole shebang. We are professionals, after all. Many of us like how small tests let us work quickly, and how larger tests inspire safety and closure. Or we may just anticipate flak during review. We are so used to these tests that often we no longer question why we write them. This can be wasteful and dangerous. Tests are a means to an end: To reduce the key risks of a project, and to get the biggest bang for the buck. This bang may not always come from the tests that standard practice has you write, or not even from tests at all. Two examples: <div style="background: #fef2cc; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> “We built a new debugging aid. We wrote unit, integration, and UI tests. We were ready to launch.” Outstanding practice. Missing the mark. Our key risks were that we'd corrupt our data or bring down our servers for the sake of a debugging aid. None of the tests addressed this, but they gave a false sense of safety and “being done”. We stopped the launch.</div> <div style="background: #fef2cc; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;"> “We wanted to turn down a feature, so we needed to alert affected users. Again we had unit and integration tests, and even one expensive end-to-end test.” Standard practice. Wasted effort. The alert was so critical it actually needed end-to-end coverage for all scenarios. But it would be live for only three releases. The cheapest effective test? Manual testing before each release.</div> A Better Approach: Risks First For every project or feature, think about testing. Brainstorm your key risks and your best options to reduce them. Do this at the start so you don't waste effort and can adapt your design. Write them down as a QA design so you can point to it in reviews and discussions. To be sure, standard practice remains a good idea in most cases (hence it’s standard). Small tests are cheap and speed up coding and maintenance, and larger tests safeguard core use-cases and integration. Just remember: Your tests are a means. The bang is what counts. It’s your job to maximize it. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Risk-Driven Testing&url=https://testing.googleblog.com/2014/05/testing-on-toilet-risk-driven-testing.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/05/testing-on-toilet-risk-driven-testing.html#comments' style='font-weight: 500; text-decoration: underline;'>1 comment</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/05/testing-on-toilet-risk-driven-testing.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht' rel='tag'> Peter Arrenbrecht </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='8299959259506460817' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/05/testing-on-toilet-effective-testing.html' itemprop='url' title='Testing on the Toilet: Effective Testing'> Testing on the Toilet: Effective Testing </a> </h2> <div class='post-header'> <div class='published'> Wednesday, May 07, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Rich Martin, Zurich This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1pb8AvYvshNRAP4x2skdd4fc0U2reCBxsMlccwEdFlFs/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Whether we are writing an individual unit test or designing a product’s entire testing process, it is important to take a step back and think about how effective are our tests at detecting and reporting bugs in our code. To be effective, there are three important qualities that every test should try to maximize: Fidelity When the code under test is broken, the test fails. A high-fidelity test is one which is very sensitive to defects in the code under test, helping to prevent bugs from creeping into the code. Maximize fidelity by ensuring that your tests cover all the paths through your code and include all relevant assertions on the expected state. Resilience A test shouldn’t fail if the code under test isn’t defective. A resilient test is one that only fails when a breaking change is made to the code under test. Refactorings and other non-breaking changes to the code under test can be made without needing to modify the test, reducing the cost of maintaining the tests. Maximize resilience by only testing the exposed API of the code under test; avoid reaching into internals. Favor stubs and fakes over mocks; don't verify interactions with dependencies unless it is that interaction that you are explicitly validating. A flaky test obviously has very low resilience. Precision When a test fails, a high-precision test tells you exactly where the defect lies. A well-written unit test can tell you exactly which line of code is at fault. Poorly written tests (especially large end-to-end tests) often exhibit very low precision, telling you that something is broken but not where. Maximize precision by keeping your tests small and tightly focused. Choose descriptive method names that convey exactly what the test is validating. For system integration tests, validate state at every boundary. These three qualities are often in tension with each other. It's easy to write a highly resilient test (the empty test, for example), but writing a test that is both highly resilient and high-fidelity is hard. As you design and write tests, use these qualities as a framework to guide your implementation. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Rich Martin, Zurich This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1pb8AvYvshNRAP4x2skdd4fc0U2reCBxsMlccwEdFlFs/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Whether we are writing an individual unit test or designing a product’s entire testing process, it is important to take a step back and think about how effective are our tests at detecting and reporting bugs in our code. To be effective, there are three important qualities that every test should try to maximize: Fidelity When the code under test is broken, the test fails. A high-fidelity test is one which is very sensitive to defects in the code under test, helping to prevent bugs from creeping into the code. Maximize fidelity by ensuring that your tests cover all the paths through your code and include all relevant assertions on the expected state. Resilience A test shouldn’t fail if the code under test isn’t defective. A resilient test is one that only fails when a breaking change is made to the code under test. Refactorings and other non-breaking changes to the code under test can be made without needing to modify the test, reducing the cost of maintaining the tests. Maximize resilience by only testing the exposed API of the code under test; avoid reaching into internals. Favor stubs and fakes over mocks; don't verify interactions with dependencies unless it is that interaction that you are explicitly validating. A flaky test obviously has very low resilience. Precision When a test fails, a high-precision test tells you exactly where the defect lies. A well-written unit test can tell you exactly which line of code is at fault. Poorly written tests (especially large end-to-end tests) often exhibit very low precision, telling you that something is broken but not where. Maximize precision by keeping your tests small and tightly focused. Choose descriptive method names that convey exactly what the test is validating. For system integration tests, validate state at every boundary. These three qualities are often in tension with each other. It's easy to write a highly resilient test (the empty test, for example), but writing a test that is both highly resilient and high-fidelity is hard. As you design and write tests, use these qualities as a framework to guide your implementation. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Effective Testing&url=https://testing.googleblog.com/2014/05/testing-on-toilet-effective-testing.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/05/testing-on-toilet-effective-testing.html#comments' style='font-weight: 500; text-decoration: underline;'>5 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/05/testing-on-toilet-effective-testing.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Rich%20Martin' rel='tag'> Rich Martin </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='4936790310537721156' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/04/testing-on-toilet-test-behaviors-not.html' itemprop='url' title='Testing on the Toilet: Test Behaviors, Not Methods'> Testing on the Toilet: Test Behaviors, Not Methods </a> </h2> <div class='post-header'> <div class='published'> Monday, April 14, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Erik Kuefler This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1W3BQgIYWZ-glc5kxvFGVE28uu4GAodAU1LU9-r4qwIs/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. After writing a method, it's easy to write just one test that verifies everything the method does. But it can be harmful to think that tests and public methods should have a 1:1 relationship. What we really want to test are behaviors, where a single method can exhibit many behaviors, and a single behavior sometimes spans across multiple methods. Let's take a look at a bad test that verifies an entire method: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void testProcessTransaction() { User user = newUserWithBalance(LOW_BALANCE_THRESHOLD.plus(dollars(2)); transactionProcessor.processTransaction( user, new Transaction("Pile of Beanie Babies", dollars(3))); assertContains("You bought a Pile of Beanie Babies", ui.getText()); assertEquals(1, user.getEmails().size()); assertEquals("Your balance is low", user.getEmails().get(0).getSubject()); } </pre> Displaying the name of the purchased item and sending an email about the balance being low are two separate behaviors, but this test looks at both of those behaviors together just because they happen to be triggered by the same method. Tests like this very often become massive and difficult to maintain over time as additional behaviors keep getting added in—eventually it will be very hard to tell which parts of the input are responsible for which assertions. The fact that the test's name is a direct mirror of the method's name is a bad sign. It's a much better idea to use separate tests to verify separate behaviors: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void testProcessTransaction_displaysNotification() { transactionProcessor.processTransaction( new User(), new Transaction("Pile of Beanie Babies")); assertContains("You bought a Pile of Beanie Babies", ui.getText()); } @Test public void testProcessTransaction_sendsEmailWhenBalanceIsLow() { User user = newUserWithBalance(LOW_BALANCE_THRESHOLD.plus(dollars(2)); transactionProcessor.processTransaction( user, new Transaction(dollars(3))); assertEquals(1, user.getEmails().size()); assertEquals("Your balance is low", user.getEmails().get(0).getSubject()); } </pre> Now, when someone adds a new behavior, they will write a new test for that behavior. Each test will remain focused and easy to understand, no matter how many behaviors are added. This will make your tests more resilient since adding new behaviors is unlikely to break the existing tests, and clearer since each test contains code to exercise only one behavior. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Erik Kuefler This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1W3BQgIYWZ-glc5kxvFGVE28uu4GAodAU1LU9-r4qwIs/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. After writing a method, it's easy to write just one test that verifies everything the method does. But it can be harmful to think that tests and public methods should have a 1:1 relationship. What we really want to test are behaviors, where a single method can exhibit many behaviors, and a single behavior sometimes spans across multiple methods. Let's take a look at a bad test that verifies an entire method: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void testProcessTransaction() { User user = newUserWithBalance(LOW_BALANCE_THRESHOLD.plus(dollars(2)); transactionProcessor.processTransaction( user, new Transaction("Pile of Beanie Babies", dollars(3))); assertContains("You bought a Pile of Beanie Babies", ui.getText()); assertEquals(1, user.getEmails().size()); assertEquals("Your balance is low", user.getEmails().get(0).getSubject()); } </pre> Displaying the name of the purchased item and sending an email about the balance being low are two separate behaviors, but this test looks at both of those behaviors together just because they happen to be triggered by the same method. Tests like this very often become massive and difficult to maintain over time as additional behaviors keep getting added in—eventually it will be very hard to tell which parts of the input are responsible for which assertions. The fact that the test's name is a direct mirror of the method's name is a bad sign. It's a much better idea to use separate tests to verify separate behaviors: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void testProcessTransaction_displaysNotification() { transactionProcessor.processTransaction( new User(), new Transaction("Pile of Beanie Babies")); assertContains("You bought a Pile of Beanie Babies", ui.getText()); } @Test public void testProcessTransaction_sendsEmailWhenBalanceIsLow() { User user = newUserWithBalance(LOW_BALANCE_THRESHOLD.plus(dollars(2)); transactionProcessor.processTransaction( user, new Transaction(dollars(3))); assertEquals(1, user.getEmails().size()); assertEquals("Your balance is low", user.getEmails().get(0).getSubject()); } </pre> Now, when someone adds a new behavior, they will write a new test for that behavior. Each test will remain focused and easy to understand, no matter how many behaviors are added. This will make your tests more resilient since adding new behaviors is unlikely to break the existing tests, and clearer since each test contains code to exercise only one behavior. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Test Behaviors, Not Methods&url=https://testing.googleblog.com/2014/04/testing-on-toilet-test-behaviors-not.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/04/testing-on-toilet-test-behaviors-not.html#comments' style='font-weight: 500; text-decoration: underline;'>6 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/04/testing-on-toilet-test-behaviors-not.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Erik%20Kuefler' rel='tag'> Erik Kuefler </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='3890024382492165319' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/04/the-real-test-driven-development.html' itemprop='url' title='The *Real* Test Driven Development'> The *Real* Test Driven Development </a> </h2> <div class='post-header'> <div class='published'> Tuesday, April 01, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <div style="text-align: center;"> Update: APRIL FOOLS!</div> <div> </div> by Kaue Silveira Here at Google, we invest heavily in development productivity research. In fact, our TDD research group now occupies nearly an entire building of the <a href="http://en.wikipedia.org/wiki/Googleplex">Googleplex</a>. The group has been working hard to minimize the development cycle time, and we’d like to share some of the amazing progress they’ve made. The Concept In the ways of old, it used to be that people wrote tests for their existing code. This was changed by TDD (Test-driven Development), where one would write the test first and then write the code to satisfy it. The TDD research group didn’t think this was enough and wanted to elevate the humble test to the next level. We are pleased to announce the Real TDD, our latest innovation in the <a href="http://en.wikipedia.org/wiki/Program_synthesis">Program Synthesis</a> field, where you write only the tests and have the computer write the code for you! The following graph shows how the number of tests created by a small feature team grew since they started using this tool towards the end of 2013. Over the last 2 quarters, more than 89% of this team’s production code was written by the tool! <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhohkeqrLo13sc_jwlb0ubNkT23aNW8eUNbp-GLtBo0iHEfrFeA1bfXS9-66N7hFPA8aKhnCzSVmfqd3kLnkldPLPxlZBA6MdjDBEXm2CjJoNj2dJ_3MRm6ekTUcFfKzi5MRBao/s1600/image.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="393" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhohkeqrLo13sc_jwlb0ubNkT23aNW8eUNbp-GLtBo0iHEfrFeA1bfXS9-66N7hFPA8aKhnCzSVmfqd3kLnkldPLPxlZBA6MdjDBEXm2CjJoNj2dJ_3MRm6ekTUcFfKzi5MRBao/s1600/image.png" width="640" /></a></div> See it in action: Test written by a Software Engineer: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">class LinkGeneratorTest(googletest.TestCase): def setUp(self): self.generator = link_generator.LinkGenerator() def testGetLinkFromIDs(self): expected = ('https://frontend.google.com/advancedSearchResults?' 's.op=ALL&s.r0.field=ID&s.r0.val=1288585+1310696+1346270+') actual = self.generator.GetLinkFromIDs(set((1346270, 1310696, 1288585))) self.assertEqual(expected, actual)</pre> Code created by our tool: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">import urllib class LinkGenerator(object): _URL = ( 'https://frontend.google.com/advancedSearchResults?' 's.op=ALL&s.r0.field=ID&s.r0.val=') def GetLinkFromIDs(self, ids): result = [] for id in sorted(ids): result.append('%s ' % id) return self._URL + urllib.quote_plus(''.join(result)) </pre> Note that the tool is smart enough to not generate the obvious implementation of returning a constant string, but instead it correctly abstracts and generalizes the relation between inputs and outputs. It becomes smarter at every use and it’s behaving more and more like a human programmer every day. We once saw a comment in the generated code that said "I need some coffee". How does it work? We’ve trained the <a href="http://en.wikipedia.org/wiki/Google_Brain">Google Brain</a> with billions of lines of open-source software to learn about coding patterns and how product code correlates with test code. Its accuracy is further improved by using <a href="http://en.wikipedia.org/wiki/Type_inference">Type Inference</a> to infer types from code and the <a href="https://www.youtube.com/watch?v=h0OkptwfX4g&feature=youtu.be&t=21m30s">Girard-Reynolds Isomorphism</a> to infer code from types. The tool runs every time your unit test is saved, and it uses the learned model to guide a backtracking search for a code snippet that satisfies all assertions in the test. It provides sub-second responses for 99.5% of the cases (as shown in the following graph), thanks to millions of pre-computed assertion-snippet pairs stored in <a href="http://en.wikipedia.org/wiki/Spanner_(database)">Spanner</a> for global low-latency access. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgPZTIW9tWScDaF70dECjIZv-1aVdWxtQiYqk9MRVMGeh5CUdrW8P5VHvRiw5ZbItvTdJ3X4KHjcA-MUzIm4356zI2ZK5yvwArDDhyLlYOXTOriNixgBhfFDbfwMTrE05yyDTBv/s1600/image+(1).png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="394" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgPZTIW9tWScDaF70dECjIZv-1aVdWxtQiYqk9MRVMGeh5CUdrW8P5VHvRiw5ZbItvTdJ3X4KHjcA-MUzIm4356zI2ZK5yvwArDDhyLlYOXTOriNixgBhfFDbfwMTrE05yyDTBv/s1600/image+(1).png" width="640" /></a></div> How can I use it? We will offer a free (rate-limited) service that everyone can use, once we have sorted out the legal issues regarding the possibility of mixing code snippets originating from open-source projects with different licenses (e.g., <a href="http://en.wikipedia.org/wiki/GNU_General_Public_License">GPL-licensed</a> tests will simply refuse to pass <a href="http://en.wikipedia.org/wiki/BSD_licenses">BSD-licensed</a> code snippets). If you would like to try our alpha release before the public launch, leave us a comment! <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> <div style="text-align: center;"> Update: APRIL FOOLS!</div> <div> </div> by Kaue Silveira Here at Google, we invest heavily in development productivity research. In fact, our TDD research group now occupies nearly an entire building of the <a href="http://en.wikipedia.org/wiki/Googleplex">Googleplex</a>. The group has been working hard to minimize the development cycle time, and we’d like to share some of the amazing progress they’ve made. The Concept In the ways of old, it used to be that people wrote tests for their existing code. This was changed by TDD (Test-driven Development), where one would write the test first and then write the code to satisfy it. The TDD research group didn’t think this was enough and wanted to elevate the humble test to the next level. We are pleased to announce the Real TDD, our latest innovation in the <a href="http://en.wikipedia.org/wiki/Program_synthesis">Program Synthesis</a> field, where you write only the tests and have the computer write the code for you! The following graph shows how the number of tests created by a small feature team grew since they started using this tool towards the end of 2013. Over the last 2 quarters, more than 89% of this team’s production code was written by the tool! <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhohkeqrLo13sc_jwlb0ubNkT23aNW8eUNbp-GLtBo0iHEfrFeA1bfXS9-66N7hFPA8aKhnCzSVmfqd3kLnkldPLPxlZBA6MdjDBEXm2CjJoNj2dJ_3MRm6ekTUcFfKzi5MRBao/s1600/image.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="393" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhohkeqrLo13sc_jwlb0ubNkT23aNW8eUNbp-GLtBo0iHEfrFeA1bfXS9-66N7hFPA8aKhnCzSVmfqd3kLnkldPLPxlZBA6MdjDBEXm2CjJoNj2dJ_3MRm6ekTUcFfKzi5MRBao/s1600/image.png" width="640" /></a></div> See it in action: Test written by a Software Engineer: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">class LinkGeneratorTest(googletest.TestCase): def setUp(self): self.generator = link_generator.LinkGenerator() def testGetLinkFromIDs(self): expected = ('https://frontend.google.com/advancedSearchResults?' 's.op=ALL&s.r0.field=ID&s.r0.val=1288585+1310696+1346270+') actual = self.generator.GetLinkFromIDs(set((1346270, 1310696, 1288585))) self.assertEqual(expected, actual)</pre> Code created by our tool: <pre style="background: #f0f0f0; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">import urllib class LinkGenerator(object): _URL = ( 'https://frontend.google.com/advancedSearchResults?' 's.op=ALL&s.r0.field=ID&s.r0.val=') def GetLinkFromIDs(self, ids): result = [] for id in sorted(ids): result.append('%s ' % id) return self._URL + urllib.quote_plus(''.join(result)) </pre> Note that the tool is smart enough to not generate the obvious implementation of returning a constant string, but instead it correctly abstracts and generalizes the relation between inputs and outputs. It becomes smarter at every use and it’s behaving more and more like a human programmer every day. We once saw a comment in the generated code that said "I need some coffee". How does it work? We’ve trained the <a href="http://en.wikipedia.org/wiki/Google_Brain">Google Brain</a> with billions of lines of open-source software to learn about coding patterns and how product code correlates with test code. Its accuracy is further improved by using <a href="http://en.wikipedia.org/wiki/Type_inference">Type Inference</a> to infer types from code and the <a href="https://www.youtube.com/watch?v=h0OkptwfX4g&feature=youtu.be&t=21m30s">Girard-Reynolds Isomorphism</a> to infer code from types. The tool runs every time your unit test is saved, and it uses the learned model to guide a backtracking search for a code snippet that satisfies all assertions in the test. It provides sub-second responses for 99.5% of the cases (as shown in the following graph), thanks to millions of pre-computed assertion-snippet pairs stored in <a href="http://en.wikipedia.org/wiki/Spanner_(database)">Spanner</a> for global low-latency access. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgPZTIW9tWScDaF70dECjIZv-1aVdWxtQiYqk9MRVMGeh5CUdrW8P5VHvRiw5ZbItvTdJ3X4KHjcA-MUzIm4356zI2ZK5yvwArDDhyLlYOXTOriNixgBhfFDbfwMTrE05yyDTBv/s1600/image+(1).png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="394" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgPZTIW9tWScDaF70dECjIZv-1aVdWxtQiYqk9MRVMGeh5CUdrW8P5VHvRiw5ZbItvTdJ3X4KHjcA-MUzIm4356zI2ZK5yvwArDDhyLlYOXTOriNixgBhfFDbfwMTrE05yyDTBv/s1600/image+(1).png" width="640" /></a></div> How can I use it? We will offer a free (rate-limited) service that everyone can use, once we have sorted out the legal issues regarding the possibility of mixing code snippets originating from open-source projects with different licenses (e.g., <a href="http://en.wikipedia.org/wiki/GNU_General_Public_License">GPL-licensed</a> tests will simply refuse to pass <a href="http://en.wikipedia.org/wiki/BSD_licenses">BSD-licensed</a> code snippets). If you would like to try our alpha release before the public launch, leave us a comment! <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:The *Real* Test Driven Development&url=https://testing.googleblog.com/2014/04/the-real-test-driven-development.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/04/the-real-test-driven-development.html#comments' style='font-weight: 500; text-decoration: underline;'>33 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/04/the-real-test-driven-development.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/April%20Fools' rel='tag'> April Fools </a> , <a class='label' href='https://testing.googleblog.com/search/label/Kaue%20Silveira' rel='tag'> Kaue Silveira </a> </div> </div> </div> <div class='post' data-id='3153681591777252403' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/03/testing-on-toilet-what-makes-good-test.html' itemprop='url' title='Testing on the Toilet: What Makes a Good Test?'> Testing on the Toilet: What Makes a Good Test? </a> </h2> <div class='post-header'> <div class='published'> Tuesday, March 18, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Erik Kuefler This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1C51MIOYAy699S5Sm2WRDsCnutUIQyi7Tzw5EbhRo_Mo/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Unit tests are important tools for verifying that our code is correct. But writing good tests is about much more than just verifying correctness — a good unit test should exhibit several other properties in order to be readable and maintainable. One property of a good test is clarity. Clarity means that a test should serve as readable documentation for humans, describing the code being tested in terms of its public APIs. Tests shouldn't refer directly to implementation details. The names of a class's tests should say everything the class does, and the tests themselves should serve as examples for how to use the class. Two more important properties are completeness and conciseness. A test is complete when its body contains all of the information you need to understand it, and concise when it doesn't contain any other distracting information. This test fails on both counts: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldPerformAddition() { Calculator calculator = new Calculator(new RoundingStrategy(), "unused", ENABLE_COSIN_FEATURE, 0.01, calculusEngine, false); int result = calculator.doComputation(makeTestComputation()); assertEquals(5, result); // Where did this number come from? }</pre> Lots of distracting information is being passed to the constructor, and the important parts are hidden off in a helper method. The test can be made more complete by clarifying the purpose of the helper method, and more concise by using another helper to hide the irrelevant details of constructing the calculator: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldPerformAddition() { Calculator calculator = newCalculator(); int result = calculator.doComputation(makeAdditionComputation(2, 3)); assertEquals(5, result); }</pre> One final property of a good test is resilience. Once written, a resilient test doesn't have to change unless the purpose or behavior of the class being tested changes. Adding new behavior should only require adding new tests, not changing old ones. The original test above isn't resilient since you'll have to update it (and probably dozens of other tests!) whenever you add a new irrelevant constructor parameter. Moving these details into the helper method solved this problem. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Erik Kuefler This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/document/d/1C51MIOYAy699S5Sm2WRDsCnutUIQyi7Tzw5EbhRo_Mo/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Unit tests are important tools for verifying that our code is correct. But writing good tests is about much more than just verifying correctness — a good unit test should exhibit several other properties in order to be readable and maintainable. One property of a good test is clarity. Clarity means that a test should serve as readable documentation for humans, describing the code being tested in terms of its public APIs. Tests shouldn't refer directly to implementation details. The names of a class's tests should say everything the class does, and the tests themselves should serve as examples for how to use the class. Two more important properties are completeness and conciseness. A test is complete when its body contains all of the information you need to understand it, and concise when it doesn't contain any other distracting information. This test fails on both counts: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldPerformAddition() { Calculator calculator = new Calculator(new RoundingStrategy(), "unused", ENABLE_COSIN_FEATURE, 0.01, calculusEngine, false); int result = calculator.doComputation(makeTestComputation()); assertEquals(5, result); // Where did this number come from? }</pre> Lots of distracting information is being passed to the constructor, and the important parts are hidden off in a helper method. The test can be made more complete by clarifying the purpose of the helper method, and more concise by using another helper to hide the irrelevant details of constructing the calculator: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">@Test public void shouldPerformAddition() { Calculator calculator = newCalculator(); int result = calculator.doComputation(makeAdditionComputation(2, 3)); assertEquals(5, result); }</pre> One final property of a good test is resilience. Once written, a resilient test doesn't have to change unless the purpose or behavior of the class being tested changes. Adding new behavior should only require adding new tests, not changing old ones. The original test above isn't resilient since you'll have to update it (and probably dozens of other tests!) whenever you add a new irrelevant constructor parameter. Moving these details into the helper method solved this problem. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: What Makes a Good Test?&url=https://testing.googleblog.com/2014/03/testing-on-toilet-what-makes-good-test.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/03/testing-on-toilet-what-makes-good-test.html#comments' style='font-weight: 500; text-decoration: underline;'>6 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/03/testing-on-toilet-what-makes-good-test.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Erik%20Kuefler' rel='tag'> Erik Kuefler </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='824867900608979117' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2014/03/whenhow-to-use-mockito-answer.html' itemprop='url' title='When/how to use Mockito Answer'> When/how to use Mockito Answer </a> </h2> <div class='post-header'> <div class='published'> Monday, March 03, 2014 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Hongfei Ding, Software Engineer, Shanghai <a href="https://code.google.com/p/mockito/">Mockito</a> is a popular open source Java testing framework that allows the creation of mock objects. For example, we have the below interface used in our SUT (System Under Test): <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">interface Service { Data get(); }</pre> In our test, normally we want to fake the Service’s behavior to return canned data, so that the unit test can focus on testing the code that interacts with the Service. We use when-return clause to stub a method. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">when(service.get()).thenReturn(cannedData);</pre> But sometimes you need mock object behavior that's too complex for when-return. An <a href="http://docs.mockito.googlecode.com/hg/latest/org/mockito/stubbing/Answer.html">Answer</a> object can be a clean way to do this once you get the syntax right. A common usage of Answer is to stub asynchronous methods that have callbacks. For example, we have mocked the interface below: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">interface Service { void get(Callback callback); }</pre> Here you’ll find that when-return is not that helpful anymore. Answer is the replacement. For example, we can emulate a success by calling the onSuccess function of the callback. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">doAnswer(new Answer<Void>() { public Void answer(InvocationOnMock invocation) { Callback callback = (Callback) invocation.getArguments()[0]; callback.onSuccess(cannedData); return null; } }).when(service).get(any(Callback.class));</pre> Answer can also be used to make smarter stubs for synchronous methods. Smarter here means the stub can return a value depending on the input, rather than canned data. It’s sometimes quite useful. For example, we have mocked the Translator interface below: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">interface Translator { String translate(String msg); }</pre> We might choose to mock Translator to return a constant string and then assert the result. However, that test is not thorough, because the input to the translator function has been ignored. To improve this, we might capture the input and do extra verification, but then we start to fall into the “<a href="http://googletesting.blogspot.tw/2013/03/testing-on-toilet-testing-state-vs.html">testing interaction rather than testing state</a>” trap. A good usage of Answer is to reverse the input message as a fake translation. So that both things are assured by checking the result string: 1) translate has been invoked, 2) the msg being translated is correct. Notice that this time we’ve used thenAnswer syntax, a twin of doAnswer, for stubbing a non-void method. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">when(translator.translate(any(String.class))).thenAnswer(reverseMsg()) ... // extracted a method to put a descriptive name private static Answer<String> reverseMsg() { return new Answer<String>() { public String answer(InvocationOnMock invocation) { return reverseString((String) invocation.getArguments()[0])); } } }</pre> Last but not least, if you find yourself writing many nontrivial Answers, you should consider using a <a href="http://googletesting.blogspot.com/2013/06/testing-on-toilet-fake-your-way-to.html">fake</a> instead. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Hongfei Ding, Software Engineer, Shanghai <a href="https://code.google.com/p/mockito/">Mockito</a> is a popular open source Java testing framework that allows the creation of mock objects. For example, we have the below interface used in our SUT (System Under Test): <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">interface Service { Data get(); }</pre> In our test, normally we want to fake the Service’s behavior to return canned data, so that the unit test can focus on testing the code that interacts with the Service. We use when-return clause to stub a method. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">when(service.get()).thenReturn(cannedData);</pre> But sometimes you need mock object behavior that's too complex for when-return. An <a href="http://docs.mockito.googlecode.com/hg/latest/org/mockito/stubbing/Answer.html">Answer</a> object can be a clean way to do this once you get the syntax right. A common usage of Answer is to stub asynchronous methods that have callbacks. For example, we have mocked the interface below: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">interface Service { void get(Callback callback); }</pre> Here you’ll find that when-return is not that helpful anymore. Answer is the replacement. For example, we can emulate a success by calling the onSuccess function of the callback. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">doAnswer(new Answer<Void>() { public Void answer(InvocationOnMock invocation) { Callback callback = (Callback) invocation.getArguments()[0]; callback.onSuccess(cannedData); return null; } }).when(service).get(any(Callback.class));</pre> Answer can also be used to make smarter stubs for synchronous methods. Smarter here means the stub can return a value depending on the input, rather than canned data. It’s sometimes quite useful. For example, we have mocked the Translator interface below: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">interface Translator { String translate(String msg); }</pre> We might choose to mock Translator to return a constant string and then assert the result. However, that test is not thorough, because the input to the translator function has been ignored. To improve this, we might capture the input and do extra verification, but then we start to fall into the “<a href="http://googletesting.blogspot.tw/2013/03/testing-on-toilet-testing-state-vs.html">testing interaction rather than testing state</a>” trap. A good usage of Answer is to reverse the input message as a fake translation. So that both things are assured by checking the result string: 1) translate has been invoked, 2) the msg being translated is correct. Notice that this time we’ve used thenAnswer syntax, a twin of doAnswer, for stubbing a non-void method. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">when(translator.translate(any(String.class))).thenAnswer(reverseMsg()) ... // extracted a method to put a descriptive name private static Answer<String> reverseMsg() { return new Answer<String>() { public String answer(InvocationOnMock invocation) { return reverseString((String) invocation.getArguments()[0])); } } }</pre> Last but not least, if you find yourself writing many nontrivial Answers, you should consider using a <a href="http://googletesting.blogspot.com/2013/06/testing-on-toilet-fake-your-way-to.html">fake</a> instead. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:When/how to use Mockito Answer&url=https://testing.googleblog.com/2014/03/whenhow-to-use-mockito-answer.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2014/03/whenhow-to-use-mockito-answer.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2014/03/whenhow-to-use-mockito-answer.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Hongfei%20Ding' rel='tag'> Hongfei Ding </a> , <a class='label' href='https://testing.googleblog.com/search/label/Java' rel='tag'> Java </a> </div> </div> </div> <div class='blog-pager' id='blog-pager'> <a class='home-link' href='https://testing.googleblog.com/'>  </a> <a class='blog-pager-newer-link' href='https://testing.googleblog.com/search?updated-max=2015-04-22T15:49:00-07:00&max-results=5&reverse-paginate=true' id='Blog1_blog-pager-newer-link' title='Newer Posts'>  </a> <a class='blog-pager-older-link' href='https://testing.googleblog.com/search?updated-max=2014-03-03T13:19:00-08:00&max-results=5' id='Blog1_blog-pager-older-link' title='Older Posts'>  </a> </div> <div class='clear'></div> </div></div> </div> </div> <div class='col-right'> <div class='section' id='sidebar-top'><div class='widget HTML' data-version='1' id='HTML8'> <div class='widget-content'> <div class='searchBox'> <input type='text' title='Search This Blog' placeholder='Search blog ...' /> </div> </div> <div class='clear'></div> </div> </div> <div id='aside'> <div class='section' id='sidebar'><div class='widget Label' data-version='1' id='Label1'> <div class='tab'> <img class='sidebar-icon' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII='/> <h2> Labels </h2>  </div> <div class='widget-content list-label-widget-content'> <ul> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/TotT'> TotT </a> 102 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/GTAC'> GTAC </a> 61 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/James%20Whittaker'> James Whittaker </a> 42 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Misko%20Hevery'> Misko Hevery </a> 32 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Code%20Health'> Code Health </a> 30 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anthony%20Vallone'> Anthony Vallone </a> 27 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrick%20Copeland'> Patrick Copeland </a> 23 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jobs'> Jobs </a> 18 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Andrew%20Trenk'> Andrew Trenk </a> 12 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/C%2B%2B'> C++ </a> 11 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund'> Patrik Höglund </a> 8 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/JavaScript'> JavaScript </a> 7 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Allen%20Hutchison'> Allen Hutchison </a> 6 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/George%20Pirocanac'> George Pirocanac </a> 6 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zhanyong%20Wan'> Zhanyong Wan </a> 6 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Harry%20Robinson'> Harry Robinson </a> 5 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Java'> Java </a> 5 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julian%20Harty'> Julian Harty </a> 5 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Bender'> Adam Bender </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alberto%20Savoia'> Alberto Savoia </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ben%20Yu'> Ben Yu </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Erik%20Kuefler'> Erik Kuefler </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Philip%20Zembrod'> Philip Zembrod </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shyam%20Seshadri'> Shyam Seshadri </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome'> Chrome </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dillon%20Bly'> Dillon Bly </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Thomas'> John Thomas </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Lesley%20Katzen'> Lesley Katzen </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Kaplan'> Marc Kaplan </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Markus%20Clermont'> Markus Clermont </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander'> Max Kanat-Alexander </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sonal%20Shah'> Sonal Shah </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/APIs'> APIs </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Abhishek%20Arya'> Abhishek Arya </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Myrvold'> Alan Myrvold </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alek%20Icev'> Alek Icev </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Android'> Android </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/April%20Fools'> April Fools </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chaitali%20Narla'> Chaitali Narla </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chris%20Lewis'> Chris Lewis </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome%20OS'> Chrome OS </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Salas'> Diego Salas </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dori%20Reuveni'> Dori Reuveni </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Arbon'> Jason Arbon </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jochen%20Wuttke'> Jochen Wuttke </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kostya%20Serebryany'> Kostya Serebryany </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Eaddy'> Marc Eaddy </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87'> Marko Ivanković </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mobile'> Mobile </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Oliver%20Chang'> Oliver Chang </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simon%20Stewart'> Simon Stewart </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stefan%20Kennedy'> Stefan Kennedy </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Flakiness'> Test Flakiness </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Titus%20Winters'> Titus Winters </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tony%20Voellm'> Tony Voellm </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/WebRTC'> WebRTC </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yiming%20Sun'> Yiming Sun </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yvette%20Nameth'> Yvette Nameth </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zuri%20Kemp'> Zuri Kemp </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Aaron%20Jacobs'> Aaron Jacobs </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Porter'> Adam Porter </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Raider'> Adam Raider </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adel%20Saoud'> Adel Saoud </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Faulkner'> Alan Faulkner </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alex%20Eagle'> Alex Eagle </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Amy%20Fu'> Amy Fu </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anantha%20Keesara'> Anantha Keesara </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Antoine%20Picard'> Antoine Picard </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/App%20Engine'> App Engine </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ari%20Shamash'> Ari Shamash </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Arif%20Sukoco'> Arif Sukoco </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Benjamin%20Pick'> Benjamin Pick </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bob%20Nystrom'> Bob Nystrom </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bruce%20Leban'> Bruce Leban </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Arguelles'> Carlos Arguelles </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Israel%20Ortiz%20Garc%C3%ADa'> Carlos Israel Ortiz García </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Cathal%20Weakliam'> Cathal Weakliam </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Christopher%20Semturs'> Christopher Semturs </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Clay%20Murphy'> Clay Murphy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dagang%20Wei'> Dagang Wei </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Maksimovich'> Dan Maksimovich </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Shi'> Dan Shi </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Willemsen'> Dan Willemsen </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Chen'> Dave Chen </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Gladfelter'> Dave Gladfelter </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Bendory'> David Bendory </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Mandelberg'> David Mandelberg </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Derek%20Snyder'> Derek Snyder </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Cavalcanti'> Diego Cavalcanti </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov'> Dmitry Vyukov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz'> Eduardo Bravo Ortiz </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ekaterina%20Kamenskaya'> Ekaterina Kamenskaya </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliott%20Karpilovsky'> Elliott Karpilovsky </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliotte%20Rusty%20Harold'> Elliotte Rusty Harold </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Espresso'> Espresso </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Felipe%20Sodr%C3%A9'> Felipe Sodré </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Francois%20Aube'> Francois Aube </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Gene%20Volovich'> Gene Volovich </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Google%2B'> Google+ </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goran%20Petrovic'> Goran Petrovic </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goranka%20Bjedov'> Goranka Bjedov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hank%20Duan'> Hank Duan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Havard%20Rast%20Blok'> Havard Rast Blok </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hongfei%20Ding'> Hongfei Ding </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Elbaum'> Jason Elbaum </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Huggins'> Jason Huggins </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jay%20Han'> Jay Han </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Hoy'> Jeff Hoy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Listfield'> Jeff Listfield </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jessica%20Tomechak'> Jessica Tomechak </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jim%20Reardon'> Jim Reardon </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joe%20Allan%20Muharsky'> Joe Allan Muharsky </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joel%20Hynoski'> Joel Hynoski </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Micco'> John Micco </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Penix'> John Penix </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway'> Jonathan Rockway </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez'> Jonathan Velasquez </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Josh%20Armour'> Josh Armour </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julie%20Ralph'> Julie Ralph </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kai%20Kent'> Kai Kent </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Karin%20Lundberg'> Karin Lundberg </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kaue%20Silveira'> Kaue Silveira </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Bourrillion'> Kevin Bourrillion </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Graney'> Kevin Graney </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kirkland'> Kirkland </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever'> Kurt Alfred Kluever </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Manjusha%20Parvathaneni'> Manjusha Parvathaneni </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis'> Marek Kiszkis </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marius%20Latinis'> Marius Latinis </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Ivey'> Mark Ivey </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Manley'> Mark Manley </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Striebeck'> Mark Striebeck </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Matt%20Lowrie'> Matt Lowrie </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Meredith%20Whittaker'> Meredith Whittaker </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Bachman'> Michael Bachman </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Klepikov'> Michael Klepikov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Aizatsky'> Mike Aizatsky </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Wacker'> Mike Wacker </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy'> Mona El Mahdy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Noel%20Yap'> Noel Yap </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Palak%20Bansal'> Palak Bansal </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patricia%20Legaspi'> Patricia Legaspi </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Per%20Jacobsson'> Per Jacobsson </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht'> Peter Arrenbrecht </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Spragins'> Peter Spragins </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Norman'> Phil Norman </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Rollet'> Phil Rollet </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Pooja%20Gupta'> Pooja Gupta </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Project%20Showcase'> Project Showcase </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Radoslav%20Vasilev'> Radoslav Vasilev </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Dewan'> Rajat Dewan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Jain'> Rajat Jain </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rich%20Martin'> Rich Martin </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Richard%20Bustamante'> Richard Bustamante </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Roshan%20Sembacuttiaratchy'> Roshan Sembacuttiaratchy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov'> Ruslan Khamitov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sam%20Lee'> Sam Lee </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sean%20Jordan'> Sean Jordan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sharon%20Zhou'> Sharon Zhou </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shiva%20Garg'> Shiva Garg </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Siddartha%20Janga'> Siddartha Janga </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simran%20Basi'> Simran Basi </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stan%20Chan'> Stan Chan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stephen%20Ng'> Stephen Ng </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tejas%20Shah'> Tejas Shah </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Analytics'> Test Analytics </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Engineer'> Test Engineer </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tim%20Lyakhovetskiy'> Tim Lyakhovetskiy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tom%20O%27Neill'> Tom O'Neill </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Vojta%20J%C3%ADna'> Vojta Jína </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/automation'> automation </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/dead%20code'> dead code </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/iOS'> iOS </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/mutation%20testing'> mutation testing </a> 1 </li> </ul> <div class='clear'></div> </div> </div><div class='widget BlogArchive' data-version='1' id='BlogArchive1'> <div class='tab'>  <h2> Archive </h2>  </div> <div class='widget-content'> <div id='ArchiveList'> <div id='BlogArchive1_ArchiveList'> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/'> 2024 </a> (12) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/04/'> Apr </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/03/'> Mar </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/02/'> Feb </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/'> 2023 </a> (14) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/12/'> Dec </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/10/'> Oct </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/09/'> Sep </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/04/'> Apr </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/'> 2022 </a> (2) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/02/'> Feb </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/'> 2021 </a> (3) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/06/'> Jun </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/04/'> Apr </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/03/'> Mar </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/'> 2020 </a> (8) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/12/'> Dec </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/08/'> Aug </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/05/'> May </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/'> 2019 </a> (4) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/01/'> Jan </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/'> 2018 </a> (7) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/05/'> May </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/02/'> Feb </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/'> 2017 </a> (17) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/07/'> Jul </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/02/'> Feb </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/'> 2016 </a> (15) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/09/'> Sep </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/04/'> Apr </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/03/'> Mar </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/02/'> Feb </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/'> 2015 </a> (14) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/10/'> Oct </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/06/'> Jun </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/03/'> Mar </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/02/'> Feb </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> ▼  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/'> 2014 </a> (24) <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> ▼  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/12/'> Dec </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/12/testing-on-toilet-truth-fluent.html'> Testing on the Toilet: Truth: a fluent assertion f... </a> </li> <li> <a href='https://testing.googleblog.com/2014/12/gtac-2014-wrap-up.html'> GTAC 2014 Wrap-up </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/11/'> Nov </a> (1) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/11/protractor-angular-testing-made-easy.html'> Protractor: Angular testing made easy </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/10/'> Oct </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/10/gtac-2014-is-this-week.html'> GTAC 2014 is this Week! </a> </li> <li> <a href='https://testing.googleblog.com/2014/10/testing-on-toilet-writing-descriptive.html'> Testing on the Toilet: Writing Descriptive Test Names </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/09/'> Sep </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/09/announcing-gtac-2014-agenda.html'> Announcing the GTAC 2014 Agenda </a> </li> <li> <a href='https://testing.googleblog.com/2014/09/chrome-firefox-webrtc-interop-test-pt-2.html'> Chrome - Firefox WebRTC Interop Test - Pt 2 </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/08/'> Aug </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/08/chrome-firefox-webrtc-interop-test-pt-1.html'> Chrome - Firefox WebRTC Interop Test - Pt 1 </a> </li> <li> <a href='https://testing.googleblog.com/2014/08/testing-on-toilet-web-testing-made.html'> Testing on the Toilet: Web Testing Made Easier: De... </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/07/'> Jul </a> (3) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/07/testing-on-toilet-dont-put-logic-in.html'> Testing on the Toilet: Don't Put Logic in Tests </a> </li> <li> <a href='https://testing.googleblog.com/2014/07/the-deadline-to-sign-up-for-gtac-2014.html'> The Deadline to Sign up for GTAC 2014 is Jul 28 </a> </li> <li> <a href='https://testing.googleblog.com/2014/07/measuring-coverage-at-google.html'> Measuring Coverage at Google </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/06/'> Jun </a> (3) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/06/threadsanitizer-slaughtering-data-races.html'> ThreadSanitizer: Slaughtering Data Races </a> </li> <li> <a href='https://testing.googleblog.com/2014/06/gtac-2014-call-for-proposals-attendance.html'> GTAC 2014: Call for Proposals & Attendance </a> </li> <li> <a href='https://testing.googleblog.com/2014/06/gtac-2014-coming-to-seattlekirkland-in.html'> GTAC 2014 Coming to Seattle/Kirkland in October </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/05/'> May </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/05/testing-on-toilet-risk-driven-testing.html'> Testing on the Toilet: Risk-Driven Testing </a> </li> <li> <a href='https://testing.googleblog.com/2014/05/testing-on-toilet-effective-testing.html'> Testing on the Toilet: Effective Testing </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/04/'> Apr </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/04/testing-on-toilet-test-behaviors-not.html'> Testing on the Toilet: Test Behaviors, Not Methods </a> </li> <li> <a href='https://testing.googleblog.com/2014/04/the-real-test-driven-development.html'> The *Real* Test Driven Development </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/03/'> Mar </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2014/03/testing-on-toilet-what-makes-good-test.html'> Testing on the Toilet: What Makes a Good Test? </a> </li> <li> <a href='https://testing.googleblog.com/2014/03/whenhow-to-use-mockito-answer.html'> When/how to use Mockito Answer </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/02/'> Feb </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/'> 2013 </a> (16) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/08/'> Aug </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/03/'> Mar </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/'> 2012 </a> (11) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/10/'> Oct </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/08/'> Aug </a> (4) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/'> 2011 </a> (39) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/10/'> Oct </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/09/'> Sep </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/08/'> Aug </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/07/'> Jul </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/06/'> Jun </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/05/'> May </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/04/'> Apr </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/03/'> Mar </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/02/'> Feb </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/01/'> Jan </a> (3) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/'> 2010 </a> (37) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/12/'> Dec </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/11/'> Nov </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/10/'> Oct </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/09/'> Sep </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/08/'> Aug </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/07/'> Jul </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/04/'> Apr </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/03/'> Mar </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/02/'> Feb </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/01/'> Jan </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/'> 2009 </a> (54) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/12/'> Dec </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/10/'> Oct </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/09/'> Sep </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/08/'> Aug </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/07/'> Jul </a> (15) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/06/'> Jun </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/02/'> Feb </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/01/'> Jan </a> (4) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/'> 2008 </a> (75) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/12/'> Dec </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/11/'> Nov </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/10/'> Oct </a> (9) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/09/'> Sep </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/08/'> Aug </a> (9) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/07/'> Jul </a> (9) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/06/'> Jun </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/05/'> May </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/04/'> Apr </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/03/'> Mar </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/02/'> Feb </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/'> 2007 </a> (41) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/10/'> Oct </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/09/'> Sep </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/08/'> Aug </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/07/'> Jul </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/04/'> Apr </a> (7) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/03/'> Mar </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/02/'> Feb </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/01/'> Jan </a> (4) </li> </ul> </li> </ul> </div> </div> <div class='clear'></div> </div> </div><div class='widget HTML' data-version='1' id='HTML6'> <div class='widget-content'> <a href="http://googletesting.blogspot.com/atom.xml"> <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==" class="sidebar-icon" /> <h2>Feed</h2> </a> </div> <div class='clear'></div> </div></div> <div class='section' id='sidebar-bottom'><div class='widget HTML' data-version='1' id='HTML9'> <div class='widget-content'> <a href='http://cloud.feedly.com/#subscription%2Ffeed%2Fhttp%3A%2F%2Fgoogletesting.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='blank'><img id="feedlyFollow" src="https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_u2h41FvEnscvnjkt-w0P6cCEH6IC_bL1okBzHRWIvHWft3BWkuIPXA74XGkQeJv2P5LwDe1loWBzz679k71OVaavTiDLgAkLSMgjdW8fHPzn7wisBWhdf0karVBVZ0yt3UUeLOSm_e2ph0NNvYDpg=s0-d" alt="follow us in feedly" width="66" height="20"></a> <div class="share followgooglewrapper"> <button data-href="https://twitter.com/intent/follow?original_referer=http://googletesting.blogspot.com/&screen_name=googletesting" onclick='sharingPopup(this);' id='twitter-share'>Follow @googletesting</button> <script> function sharingPopup (button) { var url = button.getAttribute("data-href"); window.open( url,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); } </script> </div> </div> <div class='clear'></div> </div></div> </div> </div> <div style='clear:both;'></div> </div>  <div class='google-footer-outer loading'> <div id='google-footer'> <a href='//www.google.com/'> <img class='google-logo-dark' height='36' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==' style='margin-top: -16px;' width='92'/> </a> <ul> <li> <a href='//www.google.com/'> Google </a> </li> <li> <a href='//www.google.com/policies/privacy/'> Privacy </a> </li> <li> <a href='//www.google.com/policies/terms/'> Terms </a> </li> </ul> </div> </div> <script type='text/javascript'> //<![CDATA[ // Social sharing popups. var postEl = document.getElementsByClassName('social-wrapper'); var postCount = postEl.length; for(i=0; i<postCount;i++){ postEl[i].addEventListener("click", function(event){ var postUrl = this.getAttribute("data-href"); window.open( postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); });} //]]> </script> <script type='text/javascript'> //<![CDATA[ var BreakpointHandler = function() { this.initted = false; this.isHomePage = false; this.isMobile = false; }; BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) { // Use $.trim for IE8 compatibility summaryHtml = $.trim(summaryHtml).replace(/( |\s)+$/,''); if (lastNode.nodeType == 3) { var lastChar = summaryHtml.slice(-1); if (!lastChar.match(/[.”"?]/)) { if (!lastChar.match(/[A-Za-z]/)) { summaryHtml = summaryHtml.slice(0, -1); } summaryHtml += ' ...'; } } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) { summaryHtml += ' ...'; } return summaryHtml; }; BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) { var seenWords = 0; var summaryHtml = ''; for (var i=0; i < content.childNodes.length; i++) { var node = content.childNodes[i]; var nodeText; if (node.nodeType == 1) { if (node.hasAttribute('data-about-pullquote')) { continue; } nodeText = node.textContent; if (nodeText === undefined) { // innerText for IE8 nodeText = node.innerText; } if (node.nodeName == 'DIV' || node.nodeName == 'B') { // Don't end early if we haven't seen enough words. if (seenWords < 10) { continue; } if (i > 0) { summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]); } break; } summaryHtml += node.outerHTML; } else if (node.nodeType == 3) { nodeText = node.nodeValue; summaryHtml += nodeText + ' '; } var words = nodeText.match(/\S+\s*/g); if (!words) { continue; } var remain = numWords - seenWords; if (words.length >= remain) { summaryHtml = this.finalizeSummary(summaryHtml, node); break; } seenWords += words.length; } return summaryHtml; }; BreakpointHandler.prototype.detect = function() { var match, pl = /\+/g, search = /([^&=]+)=?([^&]*)/g, decode = function (s) { return decodeURIComponent(s.replace(pl, " ")); }, query = window.location.search.substring(1); var urlParams = {}; while (match = search.exec(query)) urlParams[decode(match[1])] = decode(match[2]); this.isListPage = $('html').hasClass('list-page'); this.isMobile = urlParams['m'] === '1'; this.isHomePage = window.location.pathname == '/'; }; BreakpointHandler.prototype.initContent = function() { var self = this; $('.post').each(function(index) { var body = $(this).children('.post-body')[0]; var content = $(body).children('.post-content')[0]; $(content).addClass('post-original'); var data = $(content).children('script').html(); data = self.rewriteForSSL(data); // If exists, extract specified editor's preview. var match = data.match(/([\s\S]+?)<div data-is-preview.+?>([\s\S]+)<\/div>/m); if (match) { data = match[1]; } // Prevent big images from loading when they aren't needed. // This must be done as a pre-injection step, since image loading can't be // canceled once embedded into the DOM. if (self.isListPage && self.isMobile) { data = data.replace(/<(img|iframe) .+?>/g, ''); } // Insert template to be rendered as nodes. content.innerHTML = data; if (self.isListPage) { var summary = document.createElement('div'); $(summary).addClass('post-content'); $(summary).addClass('post-summary'); body.insertBefore(summary, content); if (match) { // Use provided summary. summary.innerHTML = match[2]; } else { // Generate a summary. // Summary generation relies on DOM, so it must occur after content is // inserted into the page. summary.innerHTML = self.generateSummaryFromContent(content, 30); } // Add read more link to summary. var titleAnchor = $(this).find('.title a')[0]; var link = titleAnchor.cloneNode(true); link.innerHTML = 'Read More'; $(link).addClass('read-more'); summary.appendChild(link); } }); // Firefox does not allow for proper styling of BR. if (navigator.userAgent.indexOf('Firefox') > -1) { $('.post-content br').replaceWith(''); } $('.loading').removeClass('loading'); }; BreakpointHandler.prototype.process = function() { if (!this.initted) { var makeInsecureImageRegex = function(hosts) { var whitelist = hosts.join('|').replace(/\./g,'\\.'); // Normal image tags, plus input images (yes, this is possible!) return new RegExp('(<(img|input)[^>]+?src=("|\'))http:\/\/(' + whitelist +')', 'g'); }; this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS); this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]); this.detect(); this.initContent(); this.initted = true; } }; BreakpointHandler.KNOWN_HTTPS_HOSTS = [ "www.google.org", "www.google.com", "services.google.com", "blogger.com", "draft.blogger.com", "www.blogger.com", "photos1.blogger.com", "photos2.blogger.com", "photos3.blogger.com", "blogblog.com", "img1.blogblog.com", "img2.blogblog.com", "www.blogblog.com", "www1.blogblog.com", "www2.blogblog.com", "0.bp.blogspot.com", "1.bp.blogspot.com", "2.bp.blogspot.com", "3.bp.blogspot.com", "4.bp.blogspot.com", "lh3.googleusercontent.com", "lh4.googleusercontent.com", "lh5.googleusercontent.com", "lh6.googleusercontent.com", "themes.googleusercontent.com", ]; BreakpointHandler.prototype.rewriteForSSL = function(html) { // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content. return html.replace(this.sslImageRegex, '$1https://$4') .replace(this.sslImageCurrentDomainRegex, '$1//$4') .replace(/(<(embed|iframe)[^>]+?src=("|'))http:\/\/([^"']*?(youtube|picasaweb\.google)\.com)/g, '$1https://$4') // Slideshow SWF takes a image host, so we need to rewrite that parameter. .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s'); }; $(document).ready(function() { var handler = new BreakpointHandler(); handler.process(); // Top-level navigation. $(".BlogArchive .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); $(".Label .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); // Blog archive year expansion. $('.BlogArchive .intervalToggle').click(function(ev) { ev.preventDefault(); if ($(this).parent().hasClass('collapsed')) { $(this).parent().removeClass('collapsed'); $(this).parent().addClass('expanded'); } else { $(this).parent().removeClass('expanded'); $(this).parent().addClass('collapsed'); } }); // Reverse order of months. $('.BlogArchive .intervalToggle + div').each(function(_, items) { var year = $(this); year.children().each(function(_, month) { year.prepend(month); }); }); // Set anchors to open in new tab. $('.post-content img').parent().each(function(_, node) { if (node.nodeName == 'A') { $(this).attr('target', '_blank'); } }); // Process search requests. $('.searchBox input').on("keypress", function(ev) { if (ev.which == 13) { window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val()); } }); }); (function($, window) { var archiveButton = $($('#sidebar .widget.BlogArchive h2')[0]); var folderIcon = $('#sidebar .widget.BlogArchive h2::after'); var archivePanel = $('#BlogArchive1'); archiveButton.click(function(e) { if (archivePanel.hasClass('archive-open')) { // It's open, so now we close it archivePanel.removeClass('archive-open'); archivePanel.addClass('archive-closed'); archiveButton.removeClass('archive-open'); archiveButton.addClass('archive-closed'); archivePanel.css('height', '60px'); } else { // It's closed, so open it archivePanel.removeClass('archive-closed'); archivePanel.addClass('archive-open'); archiveButton.removeClass('archive-closed'); archiveButton.addClass('archive-open'); archivePanel.css('height', 'auto'); folderIcon.css('content', 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEggkDzi8QVSua_XxLOnfxWE_8nQc1MAhRPBxJej91JW2ZBbKvb2TjaNf66ZoTqCrBivwRr-TLJYxFm2ZAiMd-zfbJ0TjblXGvUszWFbdRncNOL7jseiJCL9uQoqN8BM8FpesdSw/s1600/keyboard_arrow_up_grey600_24dp.png'); } }); })($, window); (function ($, window, devsite) { devsite.devsite = {}; devsite.expandLeftNav = devsite.expandLeftNav || {}; devsite.expandLeftNav.ANALYTICS_LABEL_ = 'Hamburger menu'; devsite.expandLeftNav.ANALYTICS_ACTION_OPEN_ = 'Open'; devsite.expandLeftNav.ANALYTICS_ACTION_CLOSE_ = 'Close'; devsite.expandLeftNav.lastState_ = 0; devsite.expandLeftNav.chekovEnabled_ = false; devsite.expandLeftNav.init = function () { devsite.expandLeftNav.chekovEnabled_ = true; var navElement; if (devsite.expandLeftNav.chekovEnabled_) { navElement = $('.devsite-nav-responsive'); } else { navElement = $('.devsite-section-nav-responsive'); } this.drawerWidth = parseInt(navElement.css('width')); navElement.css({ 'left': -(this.drawerWidth) }); var expandButton = $('.devsite-expand-section-nav'); expandButton.click(function () { devsite.expandLeftNav.handleNavigationOpened(); }); navElement.find('.devsite-nav-responsive-forward').click( devsite.expandLeftNav.openPanel); navElement.find('.devsite-nav-responsive-back').click( devsite.expandLeftNav.closePanel); if ($('.devsite-nav-responsive-tabs-panel + ' + '.devsite-nav-responsive-sidebar-panel').length) { devsite.expandLeftNav.openPanel(); } }; devsite.expandLeftNav.handleNavigationOpened = function (opt_noAnimate) { var mask; if (devsite.expandLeftNav.chekovEnabled_) { var nav = $('.devsite-nav-responsive'); if (opt_noAnimate) { nav.addClass('devsite-nav-responsive-no-animate'); mask = devsite.devsite.showSiteMask(0); } else { mask = devsite.devsite.showSiteMask(); } nav.addClass('devsite-nav-responsive-open'); } else { devsite.expandLeftNav.lastState_ = devsite.sticky.currentState; devsite.sticky.setState(devsite.sticky.state.COLLAPSED_HEADER); var headerHeight = devsite.sticky.getHeaderHeight() - devsite.sticky.desiredMargin; var drawerHeight = $(window).height() - headerHeight; mask = devsite.devsite.showArticleMask(); $('.devsite-section-nav-responsive').css({ 'height': drawerHeight, 'left': '0', 'top': headerHeight, 'visibility': 'visible' }).focus(); } $(mask).click(devsite.expandLeftNav.handleNavigationClosed); $(document).on('keydown.escape', function (e) { if (e.keyCode == $.ui.keyCode.ESCAPE) { devsite.expandLeftNav.handleNavigationClosed(); $(document).off('keydown.escape'); } }); }; devsite.expandLeftNav.getScrollTop = function () { return $(window).scrollTop(); }; devsite.expandLeftNav.handleNavigationClosed = function () { var nav; if (devsite.expandLeftNav.chekovEnabled_) { nav = $('.devsite-nav-responsive'); devsite.devsite.hideSiteMask(); nav.removeClass('devsite-nav-responsive-open ' + 'devsite-nav-responsive-no-animate'); } else { nav = $('.devsite-section-nav-responsive'); devsite.devsite.hideArticleMask(); devsite.sticky.setState(devsite.expandLeftNav.lastState_); nav .css('left', -(devsite.expandLeftNav.drawerWidth)) .one('transitionend', function () { nav.css('visibility', 'hidden'); }); } nav.scrollTop(0); }; devsite.expandLeftNav.openPanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel.show(); parentPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel .removeClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); parentPanel.hide(); }); setTimeout(function () { childPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); }, 1); }; devsite.expandLeftNav.closePanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); parentPanel .show() .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); setTimeout(function () { parentPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); childPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel.hide(); parentPanel.removeClass('devsite-nav-responsive-transition'); }); }, 1); }; devsite.expandLeftNav.FADE_SLOW_ = 'slow'; devsite.expandLeftNav.FADE_FAST_ = 'fast'; devsite.expandLeftNav.SITE_MASK_CSS_ = '.devsite-site-mask'; devsite.devsite.showSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_SLOW_; } devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, false, opt_animate); }; devsite.devsite.hideSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_FAST_; } devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, true, opt_animate); }; devsite.devsite.showArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_('.devsite-article-mask', false, devsite.expandLeftNav.FADE_SLOW_); }; devsite.devsite.hideArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_('.devsite-article-mask', true, devsite.expandLeftNav.FADE_FAST_); }; devsite.devsite.setMask_ = function(className, out, opt_fadeTime) { var query = $(className); if (opt_fadeTime === 0) { out ? query.hide() : query.show(); } else { out ? query.fadeOut(opt_fadeTime) : query.fadeIn(opt_fadeTime); } return $(className)[0]; }; devsite.devsite.setMouseScrollingEnabled = function(trueOrFalse) { if (trueOrFalse == true) { $('html, body').css({ 'overflow': '' }); } else { $('html, body').css({ 'overflow': 'hidden' }); } }; })($, window, devsite = {}); if (window.jQuery) { $(document).ready(function () { if (window.devsite) { devsite.expandLeftNav.init(); } }); } //]]> </script> <style> .widget ul{ line-height: 1.6 !important; } #sidebar ul li a{ color:black; line-height: 20px; } #sidebar ul li a:hover{ color:#4184F3; } </style> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY6fNnqom1rV31Kf29SUOb5liDOitQ:1732457093503';_WidgetManager._Init('//draft.blogger.com/rearrange?blogID\x3d15045980','//testing.googleblog.com/2014/','15045980'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '15045980', 'title': 'Google Testing Blog', 'url': 'https://testing.googleblog.com/2014/', 'canonicalUrl': 'https://testing.googleblog.com/2014/', 'homepageUrl': 'https://testing.googleblog.com/', 'searchUrl': 'https://testing.googleblog.com/search', 'canonicalHomepageUrl': 'https://testing.googleblog.com/', 'blogspotFaviconUrl': 'https://testing.googleblog.com/favicon.ico', 'bloggerUrl': 'https://draft.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'G-838ZCPQWM6', 'analytics4': true, 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Google Testing Blog - RSS\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://draft.blogger.com/feeds/15045980/posts/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/da8f33dd880cc4f1', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'archive', 'pageName': '2014', 'pageTitle': 'Google Testing Blog: 2014'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Google Testing Blog', 'description': '', 'url': 'https://testing.googleblog.com/2014/', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': true, 'isLabelSearch': false, 'archive': {'year': 2014, 'rangeMessage': 'Showing posts from 2014'}}}]); _WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/2646514562-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML9', 'sidebar-bottom', document.getElementById('HTML9'), {}, 'displayModeFull')); </script> </body> </html>