CINXE.COM

<!DOCTYPE html> <html class='v2 list-page' dir='ltr' itemscope='' itemtype='http://schema.org/Blog' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <title> Google Testing Blog: 2013 </title> <meta content='width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0' name='viewport'/> <meta content='IE=Edge' http-equiv='X-UA-Compatible'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='en_US' property='og:locale'/> <meta content='https://testing.googleblog.com/2013/' property='og:url'/> <meta content='Google Testing Blog' property='og:site_name'/>  <meta content='Google Testing Blog' property='og:title'/> <meta content='summary' name='twitter:card'/> <meta content='@googletesting' name='twitter:creator'/> <link href='https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic' rel='stylesheet' type='text/css'/> <link href='https://fonts.googleapis.com/icon?family=Material+Icons' rel='stylesheet'/> <script src='https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js' type='text/javascript'></script>  <style id='page-skin-1' type='text/css'></style> <style id='template-skin-1' type='text/css'></style>  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://testing.googleblog.com/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://testing.googleblog.com/2013/' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://testing.googleblog.com/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="Google Testing Blog - RSS" href="https://testing.googleblog.com/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="Google Testing Blog - Atom" href="https://www.blogger.com/feeds/15045980/posts/default" />  <meta content='https://testing.googleblog.com/2013/' property='og:url'/> <meta content='Google Testing Blog' property='og:title'/> <meta content='' property='og:description'/>  <base target='_self'/> <style> html { font-family: Roboto, sans-serif; -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; } body { padding: 0; /* This ensures that the scroll bar is always present, which is needed */ /* because content render happens after page load; otherwise the header */ /* would "bounce" in-between states. */ min-height: 150%; } h2 { font-size: 20px; } h1, h2, h3, h4, h5 { line-height: 2em; } html, h4, h5, h6 { font-size: 17px; } h3 { font-size: 18px !important; } a, a:visited { color: #4184F3; text-decoration: none; } a:focus, a:hover, a:active { text-decoration: none; } .Header { margin-top: 15px; } .Header h1 { font-size: 32px; font-weight: 300; line-height: 32px; height: 42px; } .header-inner .Header .titlewrapper { padding: 0; margin-top: 30px; } .header-inner .Header .descriptionwrapper { padding: 0; margin: 0; } .cols-wrapper { margin-top: 56px; } .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 60px; } .header-inner { height: 256px; position: relative; } html, .header-inner a { color: #212121; color: rgba(0,0,0,.87); } .header-inner .google-logo { display: inline-block; background-size: contain; z-index: 1; height: 46px; overflow: hidden; margin-top: 4px; margin-right: 8px; } .header-left { position: absolute; top: 50%; -webkit-transform: translateY(-50%); transform: translateY(-50%); margin-top: 12px; width: 100%; } .google-logo { margin-left: -4px; } #google-footer { position: relative; font-size: 13px; list-style: none; text-align: right; } #google-footer a { color: #444; } #google-footer ul { margin: 0; padding: 0; height: 144px; line-height: 144px; } #google-footer ul li { display: inline; } #google-footer ul li:before { color: #999; content: "\00b7"; font-weight: bold; margin: 5px; } #google-footer ul li:first-child:before { content: ''; } #google-footer .google-logo-dark { left: 0; margin-top: -16px; position: absolute; top: 50%; } /** Sitemap links. **/ .footer-inner-2 { font-size: 14px; padding-top: 42px; padding-bottom: 74px; } .footer-inner-2 .HTML h2 { color: #212121; color: rgba(0,0,0,.87); font-size: 14px; font-weight: 500; padding-left: 0; margin: 10px 0; } .footer-inner-2 .HTML ul { font-weight: normal; list-style: none; padding-left: 0; } .footer-inner-2 .HTML li { line-height: 24px; padding: 0; } .footer-inner-2 li a { color: rgba(65,132,243,.87); } /** Archive widget. **/ .BlogArchive { font-size: 13px; font-weight: normal; } .BlogArchive .widget-content { display: none; } .BlogArchive h2, .Label h2 { color: #4184F3; text-decoration: none; } .BlogArchive .hierarchy li { display: inline-block; } /* Specificity needed here to override widget CSS defaults. */ .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li { padding-left: 0; text-indent: 0; } .BlogArchive .intervalToggle { cursor: pointer; } .BlogArchive .expanded .intervalToggle .new-toggle { -ms-transform: rotate(180deg); transform: rotate(180deg); } .BlogArchive .new-toggle { float: right; padding-top: 3px; opacity: 0.87; } #ArchiveList { text-transform: uppercase; } #ArchiveList .expanded > ul:last-child { margin-bottom: 16px; } #ArchiveList .archivedate { width: 100%; } /* Months */ .BlogArchive .items { max-width: 150px; margin-left: -4px; } .BlogArchive .expanded .items { margin-bottom: 10px; overflow: hidden; } .BlogArchive .items > ul { float: left; height: 32px; } .BlogArchive .items a { padding: 0 4px; } .Label { font-size: 13px; font-weight: normal; } .sidebar-icon { display: inline-block; width: 24px; height: 24px; vertical-align: middle; margin-right: 12px; margin-top: -1px } .Label a { margin-right: 4px; } .Label .widget-content { display: none; } .FollowByEmail { font-size: 13px; font-weight: normal; } .FollowByEmail h2 { background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); background-repeat: no-repeat; background-position: 0 50%; text-indent: 30px; } .FollowByEmail .widget-content { display: none; } .searchBox input { border: 1px solid #eee; color: #212121; color: rgba(0,0,0,.87); font-size: 14px; padding: 8px 8px 8px 40px; width: 164px; font-family: Roboto, sans-serif; background: url("https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png") 8px center no-repeat; } .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */ color: rgba(0,0,0,.54); } .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */ color: #000; opacity: 0.54; } .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */ color: #000; opacity: 0.54; } .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */ color: #757575; } .widget-item-control { margin-top: 0px; } .section { margin: 0; padding: 0; } #sidebar-top { border: 1px solid #eee; } #sidebar-top > div { margin: 16px 0; } .widget ul { line-height: 1.6; } /*main post*/ .post { margin-bottom:30px; } #main .post .title { margin: 0; } #main .post .title a { color: #212121; color: rgba(0,0,0,.87); font-weight: normal; font-size: 24px; } #main .post .title a:hover { text-decoration:none; color:#4184F3; } .message, #main .post .post-header { margin: 0; padding: 0; } #main .post .post-header .caption, #main .post .post-header .labels-caption, #main .post .post-footer .caption, #main .post .post-footer .labels-caption { color: #444; font-weight: 500; } #main .tr-caption-container td { text-align: left; } #main .post .tr-caption { color: #757575; color: rgba(0,0,0,.54); display: block; max-width: 560px; padding-bottom: 20px; } #main .post .tr-caption-container { line-height: 24px; margin: -1px 0 0 0 !important; padding: 4px 0; text-align: left; } #main .post .post-header .published{ font-size:11px; font-weight:bold; } .post-header .publishdate { font-size: 17px; font-weight:normal; color: #757575; color: rgba(0,0,0,.54); } #main .post .post-footer{ font-size:12px; padding-bottom: 21px; } .label-footer { margin-bottom: 12px; margin-top: 12px; } .comment-img { margin-right: 16px; opacity: 0.54; vertical-align: middle; } #main .post .post-header .published { margin-bottom: 40px; margin-top: -2px; } .post .post-content { color: #212121; color: rgba(0,0,0,.87); font-size: 17px; margin: 25px 0 36px 0; line-height: 32px; word-wrap: break-word; } .post-body .post-content ul, .post-body .post-content ol { margin: 16px 0; padding: 0 48px; } .post-summary { display: none; } /* Capture section headers. */ .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b { display: inline-block; margin-bottom: 8px; margin-top: 24px; } .post-content li { line-height: 32px; } /* Override all post images/videos to left align. */ .post-content .separator, .post-content > div { text-align: left; } .post-content .separator > a, .post-content .separator > span { margin-left: 0 !important; } .post-content img { max-width: 100%; } .post-content .tr-caption-container img { margin-bottom: 12px; } .post-content iframe, .post-content embed { max-width: 100%; } .post-content .carousel-container { margin-bottom: 48px; } #main .post-content b { font-weight: 500; } /* These are the main paragraph spacing tweaks. */ #main .post-content br { content: ' '; display: block; padding: 4px; } .post-content .space { display: block; height: 8px; } .post-content iframe + .space, .post-content iframe + br { padding: 0 !important; } #main .post .jump-link { margin-bottom:10px; } .post-content img, .post-content iframe { margin: 30px 0 20px 0; } .post-content > img:first-child, .post-content > iframe:first-child { margin-top: 0; } .col-right .section { padding: 0 16px; } #aside { background:#fff; border:1px solid #eee; border-top: 0; } #aside .widget { margin:0; } #aside .widget h2, #ArchiveList .toggle + a.post-count-link { color: #212121; color: rgba(0,0,0,.87); font-weight: 400 !important; margin: 0; } #ArchiveList .toggle { float: right; } #ArchiveList .toggle .material-icons { padding-top: 4px; } #sidebar .tab { cursor: pointer; } #sidebar .tab .arrow { display: inline-block; float: right; } #sidebar .tab .icon { display: inline-block; vertical-align: top; height: 24px; width: 24px; margin-right: 13px; margin-left: -1px; margin-top: 1px; color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget-content > :first-child { padding-top: 8px; } #sidebar .active .tab .arrow { -ms-transform: rotate(180deg); transform: rotate(180deg); } #sidebar .arrow { color: #757575; color: rgba(0,0,0,.54); } #sidebar .widget h2 { font-size: 14px; line-height: 24px; display: inline-block; } #sidebar .widget .BlogArchive { padding-bottom: 8px; } #sidebar .widget { border-bottom: 1px solid #eee; box-shadow: 0px 1px 0 white; margin-bottom: 0; padding: 14px 0; min-height: 20px; } #sidebar .widget:last-child { border-bottom: none; box-shadow: none; margin-bottom: 0; } #sidebar ul { margin: 0; padding: 0; } #sidebar ul li { list-style:none; padding:0; } #sidebar ul li a { line-height: 32px; } #sidebar .archive { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC"); height: 24px; line-height: 24px; padding-left: 30px; } #sidebar .labels { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC"); height: 20px; line-height: 20px; padding-left: 30px; } #sidebar .rss a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII="); } #sidebar .subscription a { background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC"); } #sidebar-bottom { background: #f5f5f5; border-top:1px solid #eee; } #sidebar-bottom .widget { border-bottom: 1px solid #e0e0e0; padding: 15px 0; text-align: center; } #sidebar-bottom > div:last-child { border-bottom: 0; } #sidebar-bottom .text { line-height: 20px; } /* Home, forward, and backward pagination. */ .blog-pager { border-top : 1px #e0e0e0 solid; padding-top: 10px; margin-top: 15px; text-align: right !important; } #blog-pager { margin-botom: 0; margin-top: -14px; padding: 16px 0 0 0; } #blog-pager a { display: inline-block; } .blog-pager i.disabled { opacity: 0.2 !important; } .blog-pager i { color: black; margin-left: 16px; opacity: 0.54; } .blog-pager i:hover, .blog-pager i:active { opacity: 0.87; } #blog-pager-older-link, #blog-pager-newer-link { float: none; } .gplus-profile { background-color: #fafafa; border: 1px solid #eee; overflow: hidden; width: 212px; } .gplus-profile-inner { margin-left: -1px; margin-top: -1px; } /* Sidebar follow buttons. */ .followgooglewrapper { padding: 12px 0 0 0; } .loading { visibility: hidden; } .detail-page .post-footer .cmt_iframe_holder { padding-top: 40px !important; } /** Desktop **/ @media (max-width: 900px) { .col-right { display: none; } .col-main { margin-right: 0; min-width: initial; } .footer-outer { display: none; } .cols-wrapper { min-width: initial; } .google-footer-outer { background-color: #f5f5f5; } } /** Tablet **/ @media (max-width: 712px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 40px; } } /* An extra breakpoint accommodating for long blog titles. */ @media (max-width: 600px) { .header-left { height: 100%; position: initial; top: inherit; margin-top: 0; -webkit-transform: initial; transform: initial; } .header-title { margin-top: 18px; } .header-inner { height: auto; margin-bottom: 32px; margin-top: 32px; } .header-desc { margin-top: 12px; } .header-inner .google-logo { height: 40px; margin-top: 3px; } .header-inner .google-logo img { height: 42px; } .header-title h2 { font-size: 32px; line-height: 40px; } } /** Mobile/small desktop window; also landscape. **/ @media (max-width: 480px), (max-height: 480px) { .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer { padding: 0 16px; } .cols-wrapper { margin-top: 0; } .post-header .publishdate, .post .post-content { font-size: 16px; } .post .post-content { line-height: 28px; margin-bottom: 30px; } .post { margin-top: 30px; } .byline-author { display: block; font-size: 12px; line-height: 24px; margin-top: 6px; } #main .post .title a { font-weight: 500; color: #4c4c4c; color: rgba(0,0,0,.70); } #main .post .post-header { padding-bottom: 12px; } #main .post .post-header .published { margin-bottom: -8px; margin-top: 3px; } .post .read-more { display: block; margin-top: 14px; } .post .tr-caption { font-size: 12px; } #main .post .title a { font-size: 20px; line-height: 30px; } .post-content iframe { /* iframe won't keep aspect ratio when scaled down. */ max-height: 240px; } .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe { margin-left: -16px; max-width: inherit; width: calc(100% + 32px); } .post-content table, .post-content td { width: 100%; } #blog-pager { margin: 0; padding: 16px 0; } /** List page tweaks. **/ .list-page .post-original { display: none; } .list-page .post-summary { display: block; } .list-page .comment-container { display: none; } .list-page #blog-pager { padding-top: 0; border: 0; margin-top: -8px; } .list-page .label-footer { display: none; } .list-page #main .post .post-footer { border-bottom: 1px solid #eee; margin: -16px 0 0 0; padding: 0 0 20px 0; } .list-page .post .share { display: none; } /** Detail page tweaks. **/ .detail-page .post-footer .cmt_iframe_holder { padding-top: 32px !important; } .detail-page .label-footer { margin-bottom: 0; } .detail-page #main .post .post-footer { padding-bottom: 0; } .detail-page #comments { display: none; } } [data-about-pullquote], [data-is-preview], [data-about-syndication] { display: none; } </style> <noscript> <style> .loading { visibility: visible }</style> </noscript>  <script async='true' src='https://www.googletagmanager.com/gtag/js?id=G-838ZCPQWM6'></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-838ZCPQWM6'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=1da02fdc-f393-4bba-bcc4-96fe879b4516' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=15045980&zx=1da02fdc-f393-4bba-bcc4-96fe879b4516' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <script type='text/javascript'> //<![CDATA[ var axel = Math.random() + ""; var a = axel * 10000000000000; document.write('<iframe src="https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?" width="1" height="1" frameborder="0" style="display:none"></iframe>'); //]]> </script> <noscript> <img alt='' height='1' src='https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?' width='1'/> </noscript>  <div class='header-outer'> <div class='header-inner'> <div class='section' id='header'><div class='widget Header' data-version='1' id='Header1'> <div class='header-left'> <div class='header-title'> <a class='google-logo' href='https://testing.googleblog.com/'> <img height='50' src='https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png'/> </a> <a href='/.'> <h2> Testing Blog </h2> </a> </div> <div class='header-desc'> </div> </div> </div></div> </div> </div>  <div class='cols-wrapper loading'> <div class='col-main-wrapper'> <div class='col-main'> <div class='section' id='main'><div class='widget Blog' data-version='1' id='Blog1'> <div class='post' data-id='8574569812964726216' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/12/the-google-test-and-development.html' itemprop='url' title='The Google Test and Development Environment - Pt. 1: Office and Equipment'> The Google Test and Development Environment - Pt. 1: Office and Equipment </a> </h2> <div class='post-header'> <div class='published'> Friday, December 20, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> When conducting interviews, I often get questions about our workspace and engineering environment. What IDEs do you use? What programming languages are most common? What kind of tools do you have for testing? What does the workspace look like? Google is a company that is constantly pushing to improve itself. Just like software development itself, most environment improvements happen via a bottom-up approach. All engineers are responsible for fine-tuning, experimenting with, and improving our process, with a goal of eliminating barriers to creating products that amaze. Office space and engineering equipment can have a considerable impact on productivity. I’ll focus on these areas of our work environment in this first article of a series on the topic. Office layout Google is a highly collaborative workplace, so the open floor plan suits our engineering process. Project teams composed of Software Engineers (SWEs), Software Engineers in Test (SETs), and Test Engineers (TEs) all sit near each other or in large rooms together. The test-focused engineers are involved in every step of the development process, so it’s critical for them to sit with the product developers. This keeps the lines of communication open. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgNEz-s5yOfHoYZRzNS1wS1tTAV28ojF4RjyzOuJZaM-VMeg9g_lm98wE0bxRU7SlAv-0mwEDt20V3Joq5X-FsVMF5nZ6aiHVcfSqfMvRqIv3W0InHJxPq-Ww4llhifME7x8BBV/s1600/google-munich.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="265" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgNEz-s5yOfHoYZRzNS1wS1tTAV28ojF4RjyzOuJZaM-VMeg9g_lm98wE0bxRU7SlAv-0mwEDt20V3Joq5X-FsVMF5nZ6aiHVcfSqfMvRqIv3W0InHJxPq-Ww4llhifME7x8BBV/s400/google-munich.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Munich </div> The office space is far from rigid, and teams often rearrange desks to suit their preferences. The facilities team recently finished renovating a new floor in the New York City office, and after a day of engineering debates on optimal arrangements and white board diagrams, the floor was completely transformed. Besides the main office areas, there are lounge areas to which Googlers go for a change of scenery or a little peace and quiet. If you are trying to avoid becoming a casualty of The Great Foam Dart War, lounges are a great place to hide. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj39Eeapv-hfQWfdhwrE_eJqQRrrcZscdLKn52d1C7Ms6HDi2VK6mWmI1tDoH7NjsHi7pQT7rMM_2GyirEDpLF9Nti9g73O-RgY6plGTf5LRzU75QXDFyk5WKkKAHX1Fd31ecn6/s1600/dublin.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="265" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj39Eeapv-hfQWfdhwrE_eJqQRrrcZscdLKn52d1C7Ms6HDi2VK6mWmI1tDoH7NjsHi7pQT7rMM_2GyirEDpLF9Nti9g73O-RgY6plGTf5LRzU75QXDFyk5WKkKAHX1Fd31ecn6/s400/dublin.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Dublin </div> Working with remote teams Google’s worldwide headquarters is in Mountain View, CA, but it’s a very global company, and our project teams are often distributed across multiple sites. To help keep teams well connected, most of our conference rooms have video conferencing equipment. We make frequent use of this equipment for team meetings, presentations, and quick chats. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg2setl5zAdUYfRR7NKqJOiD2kke8EDXx1UKAraLcYKd4F8efPMPmkX-_3MuSQ1mNz0oso2Da0FXLa6QOCoGOT7tZvXCmz7X93nBGrLk_VLTAUIJ-u4KEGDGxhjKnSCw_u-m8nJ/s1600/boston.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="298" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg2setl5zAdUYfRR7NKqJOiD2kke8EDXx1UKAraLcYKd4F8efPMPmkX-_3MuSQ1mNz0oso2Da0FXLa6QOCoGOT7tZvXCmz7X93nBGrLk_VLTAUIJ-u4KEGDGxhjKnSCw_u-m8nJ/s400/boston.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Boston </div> What’s at your desk? All engineers get high-end machines and have easy access to data center machines for running large tasks. A new member on my team recently mentioned that his Google machine has 16 times the memory of the machine at his previous company. Most Google code runs on Linux, so the majority of development is done on Linux workstations. However, those that work on client code for Windows, OS X, or mobile, develop on relevant OSes. For displays, each engineer has a choice of either two 24 inch monitors or one 30 inch monitor. We also get our choice of laptop, picking from various models of Chromebook, MacBook, or Linux. These come in handy when going to meetings, lounges, or working remotely. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg_ukc-6eI4IC3KOU20uNSaUQQfjn3T2u__0TSi7IU-wKY8d9bYoVJGpEViIws1lAoeDJD5OURKnlePe8F03umZmXByK6ZY4-uaib9iC9dEyP9LRohh48dIQO9hFZ9-mDAMjl6e/s1600/zurich.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg_ukc-6eI4IC3KOU20uNSaUQQfjn3T2u__0TSi7IU-wKY8d9bYoVJGpEViIws1lAoeDJD5OURKnlePe8F03umZmXByK6ZY4-uaib9iC9dEyP9LRohh48dIQO9hFZ9-mDAMjl6e/s400/zurich.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Zurich </div> Thoughts? We are interested to hear your thoughts on this topic. Do you prefer an open-office layout, cubicles, or private offices? Should test teams be embedded with development teams, or should they operate separately? Do the benefits of offering engineers high-end equipment outweigh the costs? <div style="text-align: center;"> <a href="http://googletesting.blogspot.com/2014/01/the-google-test-and-development.html">(Continue to part 2)</a></div> <div style="text-align: center;"> </div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> When conducting interviews, I often get questions about our workspace and engineering environment. What IDEs do you use? What programming languages are most common? What kind of tools do you have for testing? What does the workspace look like? Google is a company that is constantly pushing to improve itself. Just like software development itself, most environment improvements happen via a bottom-up approach. All engineers are responsible for fine-tuning, experimenting with, and improving our process, with a goal of eliminating barriers to creating products that amaze. Office space and engineering equipment can have a considerable impact on productivity. I’ll focus on these areas of our work environment in this first article of a series on the topic. Office layout Google is a highly collaborative workplace, so the open floor plan suits our engineering process. Project teams composed of Software Engineers (SWEs), Software Engineers in Test (SETs), and Test Engineers (TEs) all sit near each other or in large rooms together. The test-focused engineers are involved in every step of the development process, so it’s critical for them to sit with the product developers. This keeps the lines of communication open. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgNEz-s5yOfHoYZRzNS1wS1tTAV28ojF4RjyzOuJZaM-VMeg9g_lm98wE0bxRU7SlAv-0mwEDt20V3Joq5X-FsVMF5nZ6aiHVcfSqfMvRqIv3W0InHJxPq-Ww4llhifME7x8BBV/s1600/google-munich.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="265" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgNEz-s5yOfHoYZRzNS1wS1tTAV28ojF4RjyzOuJZaM-VMeg9g_lm98wE0bxRU7SlAv-0mwEDt20V3Joq5X-FsVMF5nZ6aiHVcfSqfMvRqIv3W0InHJxPq-Ww4llhifME7x8BBV/s400/google-munich.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Munich </div> The office space is far from rigid, and teams often rearrange desks to suit their preferences. The facilities team recently finished renovating a new floor in the New York City office, and after a day of engineering debates on optimal arrangements and white board diagrams, the floor was completely transformed. Besides the main office areas, there are lounge areas to which Googlers go for a change of scenery or a little peace and quiet. If you are trying to avoid becoming a casualty of The Great Foam Dart War, lounges are a great place to hide. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj39Eeapv-hfQWfdhwrE_eJqQRrrcZscdLKn52d1C7Ms6HDi2VK6mWmI1tDoH7NjsHi7pQT7rMM_2GyirEDpLF9Nti9g73O-RgY6plGTf5LRzU75QXDFyk5WKkKAHX1Fd31ecn6/s1600/dublin.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="265" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj39Eeapv-hfQWfdhwrE_eJqQRrrcZscdLKn52d1C7Ms6HDi2VK6mWmI1tDoH7NjsHi7pQT7rMM_2GyirEDpLF9Nti9g73O-RgY6plGTf5LRzU75QXDFyk5WKkKAHX1Fd31ecn6/s400/dublin.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Dublin </div> Working with remote teams Google’s worldwide headquarters is in Mountain View, CA, but it’s a very global company, and our project teams are often distributed across multiple sites. To help keep teams well connected, most of our conference rooms have video conferencing equipment. We make frequent use of this equipment for team meetings, presentations, and quick chats. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg2setl5zAdUYfRR7NKqJOiD2kke8EDXx1UKAraLcYKd4F8efPMPmkX-_3MuSQ1mNz0oso2Da0FXLa6QOCoGOT7tZvXCmz7X93nBGrLk_VLTAUIJ-u4KEGDGxhjKnSCw_u-m8nJ/s1600/boston.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="298" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg2setl5zAdUYfRR7NKqJOiD2kke8EDXx1UKAraLcYKd4F8efPMPmkX-_3MuSQ1mNz0oso2Da0FXLa6QOCoGOT7tZvXCmz7X93nBGrLk_VLTAUIJ-u4KEGDGxhjKnSCw_u-m8nJ/s400/boston.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Boston </div> What’s at your desk? All engineers get high-end machines and have easy access to data center machines for running large tasks. A new member on my team recently mentioned that his Google machine has 16 times the memory of the machine at his previous company. Most Google code runs on Linux, so the majority of development is done on Linux workstations. However, those that work on client code for Windows, OS X, or mobile, develop on relevant OSes. For displays, each engineer has a choice of either two 24 inch monitors or one 30 inch monitor. We also get our choice of laptop, picking from various models of Chromebook, MacBook, or Linux. These come in handy when going to meetings, lounges, or working remotely. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg_ukc-6eI4IC3KOU20uNSaUQQfjn3T2u__0TSi7IU-wKY8d9bYoVJGpEViIws1lAoeDJD5OURKnlePe8F03umZmXByK6ZY4-uaib9iC9dEyP9LRohh48dIQO9hFZ9-mDAMjl6e/s1600/zurich.jpg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="266" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEg_ukc-6eI4IC3KOU20uNSaUQQfjn3T2u__0TSi7IU-wKY8d9bYoVJGpEViIws1lAoeDJD5OURKnlePe8F03umZmXByK6ZY4-uaib9iC9dEyP9LRohh48dIQO9hFZ9-mDAMjl6e/s400/zurich.jpg" width="400" /></a></div> <div style="text-align: center;"> Google Zurich </div> Thoughts? We are interested to hear your thoughts on this topic. Do you prefer an open-office layout, cubicles, or private offices? Should test teams be embedded with development teams, or should they operate separately? Do the benefits of offering engineers high-end equipment outweigh the costs? <div style="text-align: center;"> <a href="http://googletesting.blogspot.com/2014/01/the-google-test-and-development.html">(Continue to part 2)</a></div> <div style="text-align: center;"> </div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:The Google Test and Development Environment - Pt. 1: Office and Equipment&url=https://testing.googleblog.com/2013/12/the-google-test-and-development.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/12/the-google-test-and-development.html#comments' style='font-weight: 500; text-decoration: underline;'>20 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/12/the-google-test-and-development.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/Jobs' rel='tag'> Jobs </a> </div> </div> </div> <div class='post' data-id='6564788712939465881' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/11/webrtc-audio-quality-testing.html' itemprop='url' title='WebRTC Audio Quality Testing'> WebRTC Audio Quality Testing </a> </h2> <div class='post-header'> <div class='published'> Friday, November 08, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Patrik Höglund The <a href="http://www.webrtc.org/">WebRTC project</a> is all about enabling peer-to-peer video, voice and data transfer in the browser. To give our users the best possible experience we need to adapt the quality of the media to the bandwidth and processing power we have available. Our users encounter a wide variety of network conditions and run on a variety of devices, from powerful desktop machines with a wired broadband connection to laptops on WiFi to mobile phones on spotty 3G networks. We want to ensure good quality for all these use cases in our implementation in Chrome. To some extent we can do this with manual testing, but the breakneck pace of Chrome development makes it very hard to keep up (several hundred patches land every day)! Therefore, we'd like to test the quality of our video and voice transfer with an automated test. Ideally, we’d like to test for the most common network scenarios our users encounter, but to start we chose to implement a test where we have plenty of CPU and bandwidth. This article covers how we built such a test. Quality Metrics First, we must define what we want to measure. For instance, the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc&q=chrome_webrtc_vi&sq=package:chromium">WebRTC video quality test</a> uses <a href="http://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio">peak signal-to-noise ratio</a> and <a href="http://en.wikipedia.org/wiki/Structural_similarity">structural similarity</a> to measure the quality of the video (or to be more precise, how much the output video differs from the input video; see <a href="//www.youtube.com/watch?v=IbLNm3LsMaw&list=SPSIUOFhnxEiCODb8XQB-RUQ0RGNZ2yW7d">this GTAC 13 talk</a> for more details). The quality of the user experience is a subjective thing though. Arguably, one probably needs dozens of different metrics to really ensure a good user experience. For video, we would have to (at the very least) have some measure for frame rate and resolution besides correctness. To have the system send somewhat correct video frames seemed the most important though, which is why we chose the above metrics. For this test we wanted to start with a similar correctness metric, but for audio. It turns out there's an algorithm called <a href="http://en.wikipedia.org/wiki/PESQ">Perceptual Evaluation of Speech Quality</a> (PESQ) which analyzes two audio files and tell you how similar they are, while taking into account how the human ear works (so it ignores differences a normal person would not hear anyway). That's great, since we want our metrics to measure the user experience as much as possible. There are many aspects of voice transfer you could measure, such as latency (which is really important for voice calls), but for now we'll focus on measuring how much a voice audio stream gets distorted by the transfer. Feeding Audio Into WebRTC In the WebRTC case we already <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_browsertest.cc&sq=package:chromium">had a test</a> which would launch a Chrome browser, open two tabs, get the tabs talking to each other through a signaling server and set up a call on a single machine. Then we just needed to figure out how to feed a reference audio file into a WebRTC call and record what comes out on the other end. This part was actually harder than it sounds. The main WebRTC use case is that the web page acquires the user's mic through <a href="http://dev.w3.org/2011/webrtc/editor/getusermedia.html">getUserMedia</a>, sets up a PeerConnection with some remote peer and sends the audio from the mic through the connection to the peer where it is played in the peer's audio output device. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhHRCpqBiY9uTdkd8dF2B-N8m-diTyxykZ5pVOcPNkqX2vRfdsdHIYpa89WIWI0-jSgPq_1bypdjhAAshd8NZgkbOcpacWff9ULs8NIv04LfW8aOyc-ek7fzQgw6aP7jiw8uE1u/s1600/img1.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="316" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhHRCpqBiY9uTdkd8dF2B-N8m-diTyxykZ5pVOcPNkqX2vRfdsdHIYpa89WIWI0-jSgPq_1bypdjhAAshd8NZgkbOcpacWff9ULs8NIv04LfW8aOyc-ek7fzQgw6aP7jiw8uE1u/s640/img1.png" width="640" /></a></div> <div style="text-align: center;"> WebRTC calls transmit voice, video and data peer-to-peer, over the Internet. </div> But since this is an automated test, of course we could not have someone speak in a microphone every time the test runs; we had to feed in a known input file, so we had something to compare the recorded output audio against. Could we duct-tape a small stereo to the mic and play our audio file on the stereo? That's not very maintainable or reliable, not to mention annoying for anyone in the vicinity. What about some kind of fake device driver which makes a microphone-like device appear on the device level? The problem with that is that it's hard to control a driver from the userspace test program. Also, the test will be more complex and flaky, and the driver interaction will not be portable.<a href="#foot1">[1]</a> Instead, we chose to sidestep this problem. We used a solution where we load an audio file with <a href="https://dvcs.w3.org/hg/audio/raw-file/tip/webaudio/specification.html">WebAudio</a> and play that straight into the peer connection through the <a href="https://dvcs.w3.org/hg/audio/raw-file/tip/webaudio/webrtc-integration.html">WebAudio-PeerConnection integration</a>. That way we start the playing of the file from the same renderer process as the call itself, which made it a lot easier to time the start and end of the file. We still needed to be careful to avoid playing the file too early or too late, so we don't clip the audio at the start or end - that would destroy our PESQ scores! - but it turned out to be a workable approach.<a href="#foot2">[2]</a> Recording the Output Alright, so now we could get a WebRTC call set up with a known audio file with decent control of when the file starts playing. Now we had to record the output. There are a number of possible solutions. The most end-to-end way is to straight up record what the system sends to default audio out (like speakers or headphones). Alternatively, we could write a hook in our application to dump our audio as late as possible, like when we're just about to send it to the sound card. We went with the former. Our colleagues in the Chrome video stack team in Kirkland had already found that it's possible to configure a Windows or Linux machine to send the system's audio output (i.e. what plays on the speakers) to a virtual recording device. If we make that virtual recording device the default one, simply invoking SoundRecorder.exe and arecord respectively will record what the system is playing out. They found this works well if one also uses the <a href="http://sox.sourceforge.net/">sox</a> utility to eliminate silence around the actual audio content (recall we had some safety margins at both ends to ensure we record the whole input file as playing through the WebRTC call). We adopted the same approach, since it records what the user would hear, and yet uses only standard tools. This means we don't have to install additional software on the myriad machines that will run this test.<a href="#foot3">[3]</a> Analyzing Audio The only remaining step was to compare the silence-eliminated recording with the input file. When we first did this, we got a really bad score (like 2.0 out of 5.0, which means PESQ thinks it’s barely legible). This didn't seem to make sense, since both the input and recording sounded very similar. Turns out we didn’t think about the following: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li> We were comparing a full-band (24 kHz) input file to a wide-band (8 kHz) result (although both files were sampled at 48 kHz). This essentially amounted to a low pass filtering of the result file. </li> <li> Both files were in stereo, but PESQ is only mono-aware. </li> <li> The files were 32-bit, but the PESQ implementation is designed for 16 bits. </li> </ul> As you can see, it’s important to pay attention to what format arecord and SoundRecorder.exe records in, and make sure the input file is recorded in the same way. After correcting the input file and “rebasing”, we got the score up to about 4.0.<a href="#foot4">[4]</a> Thus, we ended up with an automated test that runs continously on the torrent of Chrome change lists and protects WebRTC's ability to transmit sound. You can see the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc&q=chrome_webrtc_a&sq=package:chromium">finished code here</a>. With automated tests and cleverly chosen metrics you can protect against most regressions a user would notice. If your product includes video and audio handling, such a test is a great addition to your testing mix. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhO7jQGJz2-LFp8V65EzQotJybeQvmNxrwC4fnbHH9fkOJc5G6E_4yWie0f-iE0SuqF6xeTJDnwRFLiS4l4QjRpLXZTF_LPV85aoBW1RcRnfP6VZJvvW1uI6f2lWcNIn4MctZL6/s1600/img2.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhO7jQGJz2-LFp8V65EzQotJybeQvmNxrwC4fnbHH9fkOJc5G6E_4yWie0f-iE0SuqF6xeTJDnwRFLiS4l4QjRpLXZTF_LPV85aoBW1RcRnfP6VZJvvW1uI6f2lWcNIn4MctZL6/s1600/img2.png" /></a></div> <div style="text-align: center;"> How the components of the test fit together. </div> Future work <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li> It might be possible to write a Chrome extension which dumps the audio from Chrome to a file. That way we get a simpler-to-maintain and portable solution. It would be less end-to-end but more than worth it due to the simplified maintenance and setup. Also, the recording tools we use are not perfect and add some distortion, which makes the score less accurate. </li> <li> There are other algorithms than PESQ to consider - for instance, <a href="http://en.wikipedia.org/wiki/POLQA">POLQA</a> is the successor to PESQ and is better at analyzing high-bandwidth audio signals. </li> <li> We are working on a solution which will run this test under simulated network conditions. Simulated networks combined with this test is a really powerful way to test our behavior under various packet loss and delay scenarios and ensure we deliver a good experience to all our users, not just those with great broadband connections. Stay tuned for future articles on that topic! </li> <li> Investigate feasibility of running this set-up on mobile devices. </li> </ul> <hr /> 1 It would be tolerable if the driver was just looping the input file, eliminating the need for the test to control the driver (i.e. the test doesn't have to tell the driver to start playing the file). This is actually what we do in the video quality test. It's a much better fit to take this approach on the video side since each recorded video frame is independent of the others. We can easily embed barcodes into each frame and evaluate them independently. This seems much harder for audio. We could possibly do <a href="http://en.wikipedia.org/wiki/Audio_watermark">audio watermarking</a>, or we could embed a kind of start marker (for instance, using DTMF tones) in the first two seconds of the input file and play the real content after that, and then do some fancy audio processing on the receiving end to figure out the start and end of the input audio. We chose not to pursue this approach due to its complexity. 2 Unfortunately, this also means we will not test the capturer path (which handles microphones, etc in WebRTC). This is an example of the frequent tradeoffs one has to do when designing an end-to-end test. Often we have to trade end-to-endness (how close the test is to the user experience) with robustness and simplicity of a test. It's not worth it to cover 5% more of the code if the test become unreliable or radically more expensive to maintain. Another example: A WebRTC call will generally involve two peers on different devices separated by the real-world internet. Writing such a test and making it reliable would be extremely difficult, so we make the test single-machine and hope we catch most of the bugs anyway. 3 It's important to keep the continuous build setup simple and the build machines easy to configure - otherwise you will inevitably pay a heavy price in maintenance when you try to scale your testing up. 4 When sending audio over the internet, we have to compress it since lossless audio consumes way too much bandwidth. WebRTC audio generally sounds great, but there's still compression artifacts if you listen closely (and, in fact, the recording tools are not perfect and add some distorsion as well). Given that this test is more about detecting regressions than measuring some absolute notion of quality, we'd like to downplay those artifacts. As our Kirkland colleagues found, one of the ways to do that is to "rebase" the input file. That means we start with a pristine recording, feed that through the WebRTC call and record what comes out on the other end. After manually verifying the quality, we use that as our input file for the actual test. In our case, it pushed our PESQ score up from 3 to about 4 (out of 5), which gives us a bit more sensitivity to regressions. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Patrik Höglund The <a href="http://www.webrtc.org/">WebRTC project</a> is all about enabling peer-to-peer video, voice and data transfer in the browser. To give our users the best possible experience we need to adapt the quality of the media to the bandwidth and processing power we have available. Our users encounter a wide variety of network conditions and run on a variety of devices, from powerful desktop machines with a wired broadband connection to laptops on WiFi to mobile phones on spotty 3G networks. We want to ensure good quality for all these use cases in our implementation in Chrome. To some extent we can do this with manual testing, but the breakneck pace of Chrome development makes it very hard to keep up (several hundred patches land every day)! Therefore, we'd like to test the quality of our video and voice transfer with an automated test. Ideally, we’d like to test for the most common network scenarios our users encounter, but to start we chose to implement a test where we have plenty of CPU and bandwidth. This article covers how we built such a test. Quality Metrics First, we must define what we want to measure. For instance, the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc&q=chrome_webrtc_vi&sq=package:chromium">WebRTC video quality test</a> uses <a href="http://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio">peak signal-to-noise ratio</a> and <a href="http://en.wikipedia.org/wiki/Structural_similarity">structural similarity</a> to measure the quality of the video (or to be more precise, how much the output video differs from the input video; see <a href="//www.youtube.com/watch?v=IbLNm3LsMaw&list=SPSIUOFhnxEiCODb8XQB-RUQ0RGNZ2yW7d">this GTAC 13 talk</a> for more details). The quality of the user experience is a subjective thing though. Arguably, one probably needs dozens of different metrics to really ensure a good user experience. For video, we would have to (at the very least) have some measure for frame rate and resolution besides correctness. To have the system send somewhat correct video frames seemed the most important though, which is why we chose the above metrics. For this test we wanted to start with a similar correctness metric, but for audio. It turns out there's an algorithm called <a href="http://en.wikipedia.org/wiki/PESQ">Perceptual Evaluation of Speech Quality</a> (PESQ) which analyzes two audio files and tell you how similar they are, while taking into account how the human ear works (so it ignores differences a normal person would not hear anyway). That's great, since we want our metrics to measure the user experience as much as possible. There are many aspects of voice transfer you could measure, such as latency (which is really important for voice calls), but for now we'll focus on measuring how much a voice audio stream gets distorted by the transfer. Feeding Audio Into WebRTC In the WebRTC case we already <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_browsertest.cc&sq=package:chromium">had a test</a> which would launch a Chrome browser, open two tabs, get the tabs talking to each other through a signaling server and set up a call on a single machine. Then we just needed to figure out how to feed a reference audio file into a WebRTC call and record what comes out on the other end. This part was actually harder than it sounds. The main WebRTC use case is that the web page acquires the user's mic through <a href="http://dev.w3.org/2011/webrtc/editor/getusermedia.html">getUserMedia</a>, sets up a PeerConnection with some remote peer and sends the audio from the mic through the connection to the peer where it is played in the peer's audio output device. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhHRCpqBiY9uTdkd8dF2B-N8m-diTyxykZ5pVOcPNkqX2vRfdsdHIYpa89WIWI0-jSgPq_1bypdjhAAshd8NZgkbOcpacWff9ULs8NIv04LfW8aOyc-ek7fzQgw6aP7jiw8uE1u/s1600/img1.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="316" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhHRCpqBiY9uTdkd8dF2B-N8m-diTyxykZ5pVOcPNkqX2vRfdsdHIYpa89WIWI0-jSgPq_1bypdjhAAshd8NZgkbOcpacWff9ULs8NIv04LfW8aOyc-ek7fzQgw6aP7jiw8uE1u/s640/img1.png" width="640" /></a></div> <div style="text-align: center;"> WebRTC calls transmit voice, video and data peer-to-peer, over the Internet. </div> But since this is an automated test, of course we could not have someone speak in a microphone every time the test runs; we had to feed in a known input file, so we had something to compare the recorded output audio against. Could we duct-tape a small stereo to the mic and play our audio file on the stereo? That's not very maintainable or reliable, not to mention annoying for anyone in the vicinity. What about some kind of fake device driver which makes a microphone-like device appear on the device level? The problem with that is that it's hard to control a driver from the userspace test program. Also, the test will be more complex and flaky, and the driver interaction will not be portable.<a href="#foot1">[1]</a> Instead, we chose to sidestep this problem. We used a solution where we load an audio file with <a href="https://dvcs.w3.org/hg/audio/raw-file/tip/webaudio/specification.html">WebAudio</a> and play that straight into the peer connection through the <a href="https://dvcs.w3.org/hg/audio/raw-file/tip/webaudio/webrtc-integration.html">WebAudio-PeerConnection integration</a>. That way we start the playing of the file from the same renderer process as the call itself, which made it a lot easier to time the start and end of the file. We still needed to be careful to avoid playing the file too early or too late, so we don't clip the audio at the start or end - that would destroy our PESQ scores! - but it turned out to be a workable approach.<a href="#foot2">[2]</a> Recording the Output Alright, so now we could get a WebRTC call set up with a known audio file with decent control of when the file starts playing. Now we had to record the output. There are a number of possible solutions. The most end-to-end way is to straight up record what the system sends to default audio out (like speakers or headphones). Alternatively, we could write a hook in our application to dump our audio as late as possible, like when we're just about to send it to the sound card. We went with the former. Our colleagues in the Chrome video stack team in Kirkland had already found that it's possible to configure a Windows or Linux machine to send the system's audio output (i.e. what plays on the speakers) to a virtual recording device. If we make that virtual recording device the default one, simply invoking SoundRecorder.exe and arecord respectively will record what the system is playing out. They found this works well if one also uses the <a href="http://sox.sourceforge.net/">sox</a> utility to eliminate silence around the actual audio content (recall we had some safety margins at both ends to ensure we record the whole input file as playing through the WebRTC call). We adopted the same approach, since it records what the user would hear, and yet uses only standard tools. This means we don't have to install additional software on the myriad machines that will run this test.<a href="#foot3">[3]</a> Analyzing Audio The only remaining step was to compare the silence-eliminated recording with the input file. When we first did this, we got a really bad score (like 2.0 out of 5.0, which means PESQ thinks it’s barely legible). This didn't seem to make sense, since both the input and recording sounded very similar. Turns out we didn’t think about the following: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li> We were comparing a full-band (24 kHz) input file to a wide-band (8 kHz) result (although both files were sampled at 48 kHz). This essentially amounted to a low pass filtering of the result file. </li> <li> Both files were in stereo, but PESQ is only mono-aware. </li> <li> The files were 32-bit, but the PESQ implementation is designed for 16 bits. </li> </ul> As you can see, it’s important to pay attention to what format arecord and SoundRecorder.exe records in, and make sure the input file is recorded in the same way. After correcting the input file and “rebasing”, we got the score up to about 4.0.<a href="#foot4">[4]</a> Thus, we ended up with an automated test that runs continously on the torrent of Chrome change lists and protects WebRTC's ability to transmit sound. You can see the <a href="https://code.google.com/p/chromium/codesearch#chromium/src/chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc&q=chrome_webrtc_a&sq=package:chromium">finished code here</a>. With automated tests and cleverly chosen metrics you can protect against most regressions a user would notice. If your product includes video and audio handling, such a test is a great addition to your testing mix. <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhO7jQGJz2-LFp8V65EzQotJybeQvmNxrwC4fnbHH9fkOJc5G6E_4yWie0f-iE0SuqF6xeTJDnwRFLiS4l4QjRpLXZTF_LPV85aoBW1RcRnfP6VZJvvW1uI6f2lWcNIn4MctZL6/s1600/img2.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhO7jQGJz2-LFp8V65EzQotJybeQvmNxrwC4fnbHH9fkOJc5G6E_4yWie0f-iE0SuqF6xeTJDnwRFLiS4l4QjRpLXZTF_LPV85aoBW1RcRnfP6VZJvvW1uI6f2lWcNIn4MctZL6/s1600/img2.png" /></a></div> <div style="text-align: center;"> How the components of the test fit together. </div> Future work <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li> It might be possible to write a Chrome extension which dumps the audio from Chrome to a file. That way we get a simpler-to-maintain and portable solution. It would be less end-to-end but more than worth it due to the simplified maintenance and setup. Also, the recording tools we use are not perfect and add some distortion, which makes the score less accurate. </li> <li> There are other algorithms than PESQ to consider - for instance, <a href="http://en.wikipedia.org/wiki/POLQA">POLQA</a> is the successor to PESQ and is better at analyzing high-bandwidth audio signals. </li> <li> We are working on a solution which will run this test under simulated network conditions. Simulated networks combined with this test is a really powerful way to test our behavior under various packet loss and delay scenarios and ensure we deliver a good experience to all our users, not just those with great broadband connections. Stay tuned for future articles on that topic! </li> <li> Investigate feasibility of running this set-up on mobile devices. </li> </ul> <hr /> 1 It would be tolerable if the driver was just looping the input file, eliminating the need for the test to control the driver (i.e. the test doesn't have to tell the driver to start playing the file). This is actually what we do in the video quality test. It's a much better fit to take this approach on the video side since each recorded video frame is independent of the others. We can easily embed barcodes into each frame and evaluate them independently. This seems much harder for audio. We could possibly do <a href="http://en.wikipedia.org/wiki/Audio_watermark">audio watermarking</a>, or we could embed a kind of start marker (for instance, using DTMF tones) in the first two seconds of the input file and play the real content after that, and then do some fancy audio processing on the receiving end to figure out the start and end of the input audio. We chose not to pursue this approach due to its complexity. 2 Unfortunately, this also means we will not test the capturer path (which handles microphones, etc in WebRTC). This is an example of the frequent tradeoffs one has to do when designing an end-to-end test. Often we have to trade end-to-endness (how close the test is to the user experience) with robustness and simplicity of a test. It's not worth it to cover 5% more of the code if the test become unreliable or radically more expensive to maintain. Another example: A WebRTC call will generally involve two peers on different devices separated by the real-world internet. Writing such a test and making it reliable would be extremely difficult, so we make the test single-machine and hope we catch most of the bugs anyway. 3 It's important to keep the continuous build setup simple and the build machines easy to configure - otherwise you will inevitably pay a heavy price in maintenance when you try to scale your testing up. 4 When sending audio over the internet, we have to compress it since lossless audio consumes way too much bandwidth. WebRTC audio generally sounds great, but there's still compression artifacts if you listen closely (and, in fact, the recording tools are not perfect and add some distorsion as well). Given that this test is more about detecting regressions than measuring some absolute notion of quality, we'd like to downplay those artifacts. As our Kirkland colleagues found, one of the ways to do that is to "rebase" the input file. That means we start with a pristine recording, feed that through the WebRTC call and record what comes out on the other end. After manually verifying the quality, we use that as our input file for the actual test. In our case, it pushed our PESQ score up from 3 to about 4 (out of 5), which gives us a bit more sensitivity to regressions. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:WebRTC Audio Quality Testing&url=https://testing.googleblog.com/2013/11/webrtc-audio-quality-testing.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/11/webrtc-audio-quality-testing.html#comments' style='font-weight: 500; text-decoration: underline;'>16 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/11/webrtc-audio-quality-testing.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund' rel='tag'> Patrik Höglund </a> , <a class='label' href='https://testing.googleblog.com/search/label/WebRTC' rel='tag'> WebRTC </a> </div> </div> </div> <div class='post' data-id='4779263456796444254' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/10/espresso-for-android-is-here.html' itemprop='url' title='Espresso for Android is here!'> Espresso for Android is here! </a> </h2> <div class='post-header'> <div class='published'> Friday, October 18, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> Cross-posted from the <a href="https://plus.sandbox.google.com/+AndroidDevelopers/posts">Android Developers Google+ Page</a> Earlier this year, we presented Espresso at GTAC as a solution to the UI testing problem. Today we are announcing the launch of the developer preview for Espresso! The compelling thing about developing Espresso was making it easy and fun for developers to write reliable UI tests. Espresso has a small, predictable, and easy to learn API, which is still open for customization. But most importantly - Espresso removes the need to think about the complexity of multi-threaded testing. With Espresso, you can think procedurally and write concise, beautiful, and reliable Android UI tests quickly. Espresso is now being used by over 30 applications within Google (Drive, Maps and G+, just to name a few). Starting from today, Espresso will also be available to our great developer community. We hope you will also enjoy testing your applications with Espresso and looking forward to your feedback and contributions! <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiS17ytDFusixLnzPjHU4lH95qz3SBz4zwZPrtu2Q5GUbeh1RsClpRvLcatTH8bHh8Zun9XGaUuAjIhXOoE74vxxJKrMy9kSRLHHKcVXrBll2hQY6nV-NeEqu9kJHVb4u1RP2HM/s1600/logo.jpeg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiS17ytDFusixLnzPjHU4lH95qz3SBz4zwZPrtu2Q5GUbeh1RsClpRvLcatTH8bHh8Zun9XGaUuAjIhXOoE74vxxJKrMy9kSRLHHKcVXrBll2hQY6nV-NeEqu9kJHVb4u1RP2HM/s1600/logo.jpeg" /></a></div> <div style="text-align: center;"> Android Test Kit: <a href="https://code.google.com/p/android-test-kit/">https://code.google.com/p/android-test-kit/</a></div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> Cross-posted from the <a href="https://plus.sandbox.google.com/+AndroidDevelopers/posts">Android Developers Google+ Page</a> Earlier this year, we presented Espresso at GTAC as a solution to the UI testing problem. Today we are announcing the launch of the developer preview for Espresso! The compelling thing about developing Espresso was making it easy and fun for developers to write reliable UI tests. Espresso has a small, predictable, and easy to learn API, which is still open for customization. But most importantly - Espresso removes the need to think about the complexity of multi-threaded testing. With Espresso, you can think procedurally and write concise, beautiful, and reliable Android UI tests quickly. Espresso is now being used by over 30 applications within Google (Drive, Maps and G+, just to name a few). Starting from today, Espresso will also be available to our great developer community. We hope you will also enjoy testing your applications with Espresso and looking forward to your feedback and contributions! <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiS17ytDFusixLnzPjHU4lH95qz3SBz4zwZPrtu2Q5GUbeh1RsClpRvLcatTH8bHh8Zun9XGaUuAjIhXOoE74vxxJKrMy9kSRLHHKcVXrBll2hQY6nV-NeEqu9kJHVb4u1RP2HM/s1600/logo.jpeg" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiS17ytDFusixLnzPjHU4lH95qz3SBz4zwZPrtu2Q5GUbeh1RsClpRvLcatTH8bHh8Zun9XGaUuAjIhXOoE74vxxJKrMy9kSRLHHKcVXrBll2hQY6nV-NeEqu9kJHVb4u1RP2HM/s1600/logo.jpeg" /></a></div> <div style="text-align: center;"> Android Test Kit: <a href="https://code.google.com/p/android-test-kit/">https://code.google.com/p/android-test-kit/</a></div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Espresso for Android is here!&url=https://testing.googleblog.com/2013/10/espresso-for-android-is-here.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/10/espresso-for-android-is-here.html#comments' style='font-weight: 500; text-decoration: underline;'>14 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/10/espresso-for-android-is-here.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Android' rel='tag'> Android </a> , <a class='label' href='https://testing.googleblog.com/search/label/Espresso' rel='tag'> Espresso </a> </div> </div> </div> <div class='post' data-id='4457464545625398291' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/08/how-google-team-tests-mobile-apps.html' itemprop='url' title='How the Google+ Team Tests Mobile Apps'> How the Google+ Team Tests Mobile Apps </a> </h2> <div class='post-header'> <div class='published'> Friday, August 30, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by Eduardo Bravo Ortiz “Mobile first” is the motto these days for many companies. However, being able to test a mobile app in a meaningful way is very challenging. On the Google+ team we have had our share of trial and error that has led us to successful strategies for testing mobile applications on both iOS and Android. General <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Understand the platform. Testing on Android is not the same as testing on iOS. The testing tools and frameworks available for each platform are significantly different. (e.g., Android uses Java while iOS uses Objective-C, UI layouts are built differently on each platform, UI testing frameworks also work very differently in both platforms.) </li> <li>Stabilize your test suite and test environments. Flaky tests are worse than having no tests, because a flaky test pollutes your build health and decreases the credibility of your suite. </li> <li> Break down testing into manageable pieces. There are too many complex pieces when testing on mobile (e.g., emulator/device state, actions triggered by the OS). </li> <li>Provide a <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">hermetic test</a> environment for your tests. Mobile UI tests are flaky by nature; don’t add more flakiness to them by having external dependencies. </li> <li>Unit tests are the backbone of your mobile test strategy. Try to separate the app code logic from the UI as much as possible. This separation will make unit tests more granular and faster. </li> </ul> Android Testing Unit Tests Separating UI code from code logic is especially hard in Android. For example, an Activity is expected to act as a controller and view at the same time; make sure you keep this in mind when writing unit tests. Another useful recommendation is to decouple unit tests from the Android emulator, this will remove the need to build an APK and install it and your tests will run much faster. <a href="http://pivotal.github.io/robolectric/">Robolectric</a> is a perfect tool for this; it stubs the implementation of the Android platform while running tests. Hermetic UI Tests A hermetic UI test typically runs as a test without network calls or external dependencies. Once the tests can run in a <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">hermetic environment</a>, a white box testing framework like <a href="//www.youtube.com/watch?v=T7ugmCuNxDU">Espresso</a> can simulate user actions on the UI and is tightly coupled to the app code. Espresso will also synchronize your tests actions with events on the UI thread, reducing flakiness. More information on Espresso is coming in a future Google Testing Blog article. <div style="text-align: center;"> Diagram: Non-Hermetic Flow vs. Hermetic Flow </div> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTTK0s_YqmAsDe4nh01dZ7QwZ8TuKP38mUOls_erCNP1lFYHwsIbzq9F_zjD2CE1k1KpIb9LOEaFFNdNwSq5db-hbrTbW6G15vFy8HHg01Q4aQ__UCuIwMNJ9Nd8yJTkoAA42C/s1600/diagram1.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="199" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTTK0s_YqmAsDe4nh01dZ7QwZ8TuKP38mUOls_erCNP1lFYHwsIbzq9F_zjD2CE1k1KpIb9LOEaFFNdNwSq5db-hbrTbW6G15vFy8HHg01Q4aQ__UCuIwMNJ9Nd8yJTkoAA42C/s640/diagram1.png" width="640" /></a></div> Monkey Tests Monkey tests look for crashes and <a href="http://developer.android.com/training/articles/perf-anr.html">ANRs</a> by stressing your Android application. They exercise <a href="http://en.wikipedia.org/wiki/Pseudorandom_number_generator">pseudo-random</a> events like clicks or gestures on the app under test. Monkey test results are reproducible to a certain extent; timing and latency are not completely under your control and can cause a test failure. Re-running the same monkey test against the same configuration will often reproduce these failures, though. If you run them daily against different SDKs, they are very effective at catching bugs earlier in the development cycle of a new release. iOS Testing Unit Tests Unit test frameworks like <a href="http://www.sente.ch/software/ocunit/">OCUnit</a>, which comes bundled with Xcode, or <a href="https://code.google.com/p/google-toolbox-for-mac/">GTMSenTestcase</a> are both good choices. Hermetic UI Tests <a href="https://github.com/kif-framework/KIF">KIF</a> has proven to be a powerful solution for writing Objective-C UI tests. It runs in-process which allows tests to be more tightly coupled with the app under test, making the tests inherently more stable. KIF allows iOS developers to write tests using the same language as their application. Following the same paradigm as Android UI tests, you want Objective-C tests to be hermetic. A good approach is to mock the server with pre-canned responses. Since KIF tests run in-process, responses can be built programmatically, making tests easier to maintain and more stable. Monkey Tests iOS has no equivalent native tool for writing monkey tests as Android does, however this type of test still adds value in iOS (e.g. we found 16 crashes in one of our recent Google+ releases). The Google+ team developed their own custom monkey testing framework, but there are also many third-party options available. Backend Testing A mobile testing strategy is not complete without testing the integration between server backends and mobile clients. This is especially true when the release cycles of the mobile clients and backends are very different. A replay test strategy can be very effective at preventing backends from breaking mobile clients. The theory behind this strategy is to simulate mobile clients by having a set of golden request and response files that are known to be correct. The replay test suite should then send golden requests to the backend server and assert that the response returned by the server matches the expected golden response. Since client/server responses are often not completely deterministic, you will need to utilize a diffing tool that can ignore expected differences. To make this strategy successful you need a way to seed a repeatable data set on the backend and make all dependencies that are not relevant to your backend hermetic. Using in-memory servers with fake data or an RPC replay to external dependencies are good ways of achieving repeatable data sets and hermetic environments. Google+ mobile backend uses <a href="https://code.google.com/p/google-guice/">Guice</a> for dependency injection, which allows us to easily swap out dependencies with fake implementations during testing and seed data fixtures. <div style="text-align: center;"> Diagram: Normal flow vs Replay Tests flow </div> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiletPFOrnC3SIgjtkgUeW1usJEqkP20AI_3aY1nsopKqmRhlk3_LcbE4WgFhQkry3J9QGS9M90invB1uA2AAymcUQjOz5hzY0dod3S75JFfmNYtXJChW_2gg1lYpTXEQXV03jM/s1600/diagram2.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="230" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiletPFOrnC3SIgjtkgUeW1usJEqkP20AI_3aY1nsopKqmRhlk3_LcbE4WgFhQkry3J9QGS9M90invB1uA2AAymcUQjOz5hzY0dod3S75JFfmNYtXJChW_2gg1lYpTXEQXV03jM/s640/diagram2.png" width="640" /></a></div> Conclusion Mobile app testing can be very challenging, but building a comprehensive test strategy that understands the nature of different platforms and tools is the key to success. Providing a reliable and hermetic test environment is as important as the tests you write. Finally, make sure you prioritize your automation efforts according to your team needs. This is how we prioritize on the Google+ team: <ol style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li> Unit tests: These should be your first priority in either Android or iOS. They run fast and are less flaky than any other type of tests. </li> <li> Backend tests: Make sure your backend doesn’t break your mobile clients. Breakages are very likely to happen when the release cycle of mobile clients and backends are different. </li> <li> UI tests: These are slower by nature and flaky. They also take more time to write and maintain. Make sure you provide coverage for at least the critical paths of your app. </li> <li> Monkey tests: This is the final step to complete your mobile automation strategy. </li> </ol> Happy mobile testing from the Google+ team. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by Eduardo Bravo Ortiz “Mobile first” is the motto these days for many companies. However, being able to test a mobile app in a meaningful way is very challenging. On the Google+ team we have had our share of trial and error that has led us to successful strategies for testing mobile applications on both iOS and Android. General <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Understand the platform. Testing on Android is not the same as testing on iOS. The testing tools and frameworks available for each platform are significantly different. (e.g., Android uses Java while iOS uses Objective-C, UI layouts are built differently on each platform, UI testing frameworks also work very differently in both platforms.) </li> <li>Stabilize your test suite and test environments. Flaky tests are worse than having no tests, because a flaky test pollutes your build health and decreases the credibility of your suite. </li> <li> Break down testing into manageable pieces. There are too many complex pieces when testing on mobile (e.g., emulator/device state, actions triggered by the OS). </li> <li>Provide a <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">hermetic test</a> environment for your tests. Mobile UI tests are flaky by nature; don’t add more flakiness to them by having external dependencies. </li> <li>Unit tests are the backbone of your mobile test strategy. Try to separate the app code logic from the UI as much as possible. This separation will make unit tests more granular and faster. </li> </ul> Android Testing Unit Tests Separating UI code from code logic is especially hard in Android. For example, an Activity is expected to act as a controller and view at the same time; make sure you keep this in mind when writing unit tests. Another useful recommendation is to decouple unit tests from the Android emulator, this will remove the need to build an APK and install it and your tests will run much faster. <a href="http://pivotal.github.io/robolectric/">Robolectric</a> is a perfect tool for this; it stubs the implementation of the Android platform while running tests. Hermetic UI Tests A hermetic UI test typically runs as a test without network calls or external dependencies. Once the tests can run in a <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">hermetic environment</a>, a white box testing framework like <a href="//www.youtube.com/watch?v=T7ugmCuNxDU">Espresso</a> can simulate user actions on the UI and is tightly coupled to the app code. Espresso will also synchronize your tests actions with events on the UI thread, reducing flakiness. More information on Espresso is coming in a future Google Testing Blog article. <div style="text-align: center;"> Diagram: Non-Hermetic Flow vs. Hermetic Flow </div> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTTK0s_YqmAsDe4nh01dZ7QwZ8TuKP38mUOls_erCNP1lFYHwsIbzq9F_zjD2CE1k1KpIb9LOEaFFNdNwSq5db-hbrTbW6G15vFy8HHg01Q4aQ__UCuIwMNJ9Nd8yJTkoAA42C/s1600/diagram1.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="199" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgTTK0s_YqmAsDe4nh01dZ7QwZ8TuKP38mUOls_erCNP1lFYHwsIbzq9F_zjD2CE1k1KpIb9LOEaFFNdNwSq5db-hbrTbW6G15vFy8HHg01Q4aQ__UCuIwMNJ9Nd8yJTkoAA42C/s640/diagram1.png" width="640" /></a></div> Monkey Tests Monkey tests look for crashes and <a href="http://developer.android.com/training/articles/perf-anr.html">ANRs</a> by stressing your Android application. They exercise <a href="http://en.wikipedia.org/wiki/Pseudorandom_number_generator">pseudo-random</a> events like clicks or gestures on the app under test. Monkey test results are reproducible to a certain extent; timing and latency are not completely under your control and can cause a test failure. Re-running the same monkey test against the same configuration will often reproduce these failures, though. If you run them daily against different SDKs, they are very effective at catching bugs earlier in the development cycle of a new release. iOS Testing Unit Tests Unit test frameworks like <a href="http://www.sente.ch/software/ocunit/">OCUnit</a>, which comes bundled with Xcode, or <a href="https://code.google.com/p/google-toolbox-for-mac/">GTMSenTestcase</a> are both good choices. Hermetic UI Tests <a href="https://github.com/kif-framework/KIF">KIF</a> has proven to be a powerful solution for writing Objective-C UI tests. It runs in-process which allows tests to be more tightly coupled with the app under test, making the tests inherently more stable. KIF allows iOS developers to write tests using the same language as their application. Following the same paradigm as Android UI tests, you want Objective-C tests to be hermetic. A good approach is to mock the server with pre-canned responses. Since KIF tests run in-process, responses can be built programmatically, making tests easier to maintain and more stable. Monkey Tests iOS has no equivalent native tool for writing monkey tests as Android does, however this type of test still adds value in iOS (e.g. we found 16 crashes in one of our recent Google+ releases). The Google+ team developed their own custom monkey testing framework, but there are also many third-party options available. Backend Testing A mobile testing strategy is not complete without testing the integration between server backends and mobile clients. This is especially true when the release cycles of the mobile clients and backends are very different. A replay test strategy can be very effective at preventing backends from breaking mobile clients. The theory behind this strategy is to simulate mobile clients by having a set of golden request and response files that are known to be correct. The replay test suite should then send golden requests to the backend server and assert that the response returned by the server matches the expected golden response. Since client/server responses are often not completely deterministic, you will need to utilize a diffing tool that can ignore expected differences. To make this strategy successful you need a way to seed a repeatable data set on the backend and make all dependencies that are not relevant to your backend hermetic. Using in-memory servers with fake data or an RPC replay to external dependencies are good ways of achieving repeatable data sets and hermetic environments. Google+ mobile backend uses <a href="https://code.google.com/p/google-guice/">Guice</a> for dependency injection, which allows us to easily swap out dependencies with fake implementations during testing and seed data fixtures. <div style="text-align: center;"> Diagram: Normal flow vs Replay Tests flow </div> <div class="separator" style="clear: both; text-align: center;"> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiletPFOrnC3SIgjtkgUeW1usJEqkP20AI_3aY1nsopKqmRhlk3_LcbE4WgFhQkry3J9QGS9M90invB1uA2AAymcUQjOz5hzY0dod3S75JFfmNYtXJChW_2gg1lYpTXEQXV03jM/s1600/diagram2.png" imageanchor="1" style="margin-left: 1em; margin-right: 1em;"><img border="0" height="230" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiletPFOrnC3SIgjtkgUeW1usJEqkP20AI_3aY1nsopKqmRhlk3_LcbE4WgFhQkry3J9QGS9M90invB1uA2AAymcUQjOz5hzY0dod3S75JFfmNYtXJChW_2gg1lYpTXEQXV03jM/s640/diagram2.png" width="640" /></a></div> Conclusion Mobile app testing can be very challenging, but building a comprehensive test strategy that understands the nature of different platforms and tools is the key to success. Providing a reliable and hermetic test environment is as important as the tests you write. Finally, make sure you prioritize your automation efforts according to your team needs. This is how we prioritize on the Google+ team: <ol style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li> Unit tests: These should be your first priority in either Android or iOS. They run fast and are less flaky than any other type of tests. </li> <li> Backend tests: Make sure your backend doesn’t break your mobile clients. Breakages are very likely to happen when the release cycle of mobile clients and backends are different. </li> <li> UI tests: These are slower by nature and flaky. They also take more time to write and maintain. Make sure you provide coverage for at least the critical paths of your app. </li> <li> Monkey tests: This is the final step to complete your mobile automation strategy. </li> </ol> Happy mobile testing from the Google+ team. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:How the Google+ Team Tests Mobile Apps&url=https://testing.googleblog.com/2013/08/how-google-team-tests-mobile-apps.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/08/how-google-team-tests-mobile-apps.html#comments' style='font-weight: 500; text-decoration: underline;'>17 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/08/how-google-team-tests-mobile-apps.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz' rel='tag'> Eduardo Bravo Ortiz </a> , <a class='label' href='https://testing.googleblog.com/search/label/Google%2B' rel='tag'> Google+ </a> , <a class='label' href='https://testing.googleblog.com/search/label/Mobile' rel='tag'> Mobile </a> </div> </div> </div> <div class='post' data-id='7474927478932637543' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/08/testing-on-toilet-test-behavior-not.html' itemprop='url' title='Testing on the Toilet: Test Behavior, Not Implementation'> Testing on the Toilet: Test Behavior, Not Implementation </a> </h2> <div class='post-header'> <div class='published'> Monday, August 05, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/file/d/0B_EQS8-2f7m5RXJvRzFIcUtMVjg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Your trusty Calculator class is one of your most popular open source projects, with many happy users: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public class Calculator { public int add(int a, int b) { return a + b; } }</pre> You also have tests to help ensure that it works properly: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void testAdd() { assertEquals(3, calculator.add(2, 1)); assertEquals(2, calculator.add(2, 0)); assertEquals(1, calculator.add(2, -1)); }</pre> However, a fancy new library promises several orders of magnitude speedup in your code if you use it in place of the addition operator. You excitedly change your code to use this library: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public class Calculator { private AdderFactory adderFactory; public Calculator(AdderFactor adderFactory) { this.adderFactory = adderFactory; } public int add(int a, int b) { Adder adder = adderFactory.createAdder(); ReturnValue returnValue = adder.compute(new Number(a), new Number(b)); return returnValue.convertToInteger(); } }</pre> That was easy, but what do you do about the tests for this code? None of the existing tests should need to change since you only changed the code's implementation, but its user-facing behavior didn't change. In most cases, tests should focus on testing your code's public API, and your code's implementation details shouldn't need to be exposed to tests. Tests that are independent of implementation details are easier to maintain since they don't need to be changed each time you make a change to the implementation. They're also easier to understand since they basically act as code samples that show all the different ways your class's methods can be used, so even someone who's not familiar with the implementation should usually be able to read through the tests to understand how to use the class. There are many cases where you do want to test implementation details (e.g. you want to ensure that your implementation reads from a cache instead of from a datastore), but this should be less common since in most cases your tests should be independent of your implementation. Note that test setup may need to change if the implementation changes (e.g. if you change your class to take a new dependency in its constructor, the test needs to pass in this dependency when it creates the class), but the actual test itself typically shouldn't need to change if the code's user-facing behavior doesn't change. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/file/d/0B_EQS8-2f7m5RXJvRzFIcUtMVjg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. Your trusty Calculator class is one of your most popular open source projects, with many happy users: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public class Calculator { public int add(int a, int b) { return a + b; } }</pre> You also have tests to help ensure that it works properly: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void testAdd() { assertEquals(3, calculator.add(2, 1)); assertEquals(2, calculator.add(2, 0)); assertEquals(1, calculator.add(2, -1)); }</pre> However, a fancy new library promises several orders of magnitude speedup in your code if you use it in place of the addition operator. You excitedly change your code to use this library: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public class Calculator { private AdderFactory adderFactory; public Calculator(AdderFactor adderFactory) { this.adderFactory = adderFactory; } public int add(int a, int b) { Adder adder = adderFactory.createAdder(); ReturnValue returnValue = adder.compute(new Number(a), new Number(b)); return returnValue.convertToInteger(); } }</pre> That was easy, but what do you do about the tests for this code? None of the existing tests should need to change since you only changed the code's implementation, but its user-facing behavior didn't change. In most cases, tests should focus on testing your code's public API, and your code's implementation details shouldn't need to be exposed to tests. Tests that are independent of implementation details are easier to maintain since they don't need to be changed each time you make a change to the implementation. They're also easier to understand since they basically act as code samples that show all the different ways your class's methods can be used, so even someone who's not familiar with the implementation should usually be able to read through the tests to understand how to use the class. There are many cases where you do want to test implementation details (e.g. you want to ensure that your implementation reads from a cache instead of from a datastore), but this should be less common since in most cases your tests should be independent of your implementation. Note that test setup may need to change if the implementation changes (e.g. if you change your class to take a new dependency in its constructor, the test needs to pass in this dependency when it creates the class), but the actual test itself typically shouldn't need to change if the code's user-facing behavior doesn't change. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Test Behavior, Not Implementation&url=https://testing.googleblog.com/2013/08/testing-on-toilet-test-behavior-not.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/08/testing-on-toilet-test-behavior-not.html#comments' style='font-weight: 500; text-decoration: underline;'>4 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/08/testing-on-toilet-test-behavior-not.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='3043935751548309619' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/07/testing-on-toilet-know-your-test-doubles.html' itemprop='url' title='Testing on the Toilet: Know Your Test Doubles'> Testing on the Toilet: Know Your Test Doubles </a> </h2> <div class='post-header'> <div class='published'> Thursday, July 18, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/file/d/0B_EQS8-2f7m5Y1lTcDRnekk3Rjg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. A test double is an object that can stand in for a real object in a test, similar to how a stunt double stands in for an actor in a movie. These are sometimes all commonly referred to as “mocks”, but it's important to distinguish between the different types of test doubles since they all have different uses. The most common types of test doubles are stubs, mocks, and fakes. A stub has no logic, and only returns what you tell it to return. Stubs can be used when you need an object to return specific values in order to get your code under test into a certain state. While it's usually easy to write stubs by hand, using a mocking framework is often a convenient way to reduce boilerplate. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// Pass in a stub that was created by a mocking framework. AccessManager accessManager = new AccessManager(stubAuthenticationService); // The user shouldn't have access when the authentication service returns false. when(stubAuthenticationService.isAuthenticated(USER_ID)).thenReturn(false); assertFalse(accessManager.userHasAccess(USER_ID)); // The user should have access when the authentication service returns true. when(stubAuthenticationService.isAuthenticated(USER_ID)).thenReturn(true); assertTrue(accessManager.userHasAccess(USER_ID)); </pre> A mock has expectations about the way it should be called, and a test should fail if it’s not called that way. Mocks are used to test interactions between objects, and are useful in cases where there are no other visible state changes or return results that you can verify (e.g. if your code reads from disk and you want to ensure that it doesn't do more than one disk read, you can use a mock to verify that the method that does the read is only called once). <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// Pass in a mock that was created by a mocking framework. AccessManager accessManager = new AccessManager(mockAuthenticationService); accessManager.userHasAccess(USER_ID); // The test should fail if accessManager.userHasAccess(USER_ID) didn't call // mockAuthenticationService.isAuthenticated(USER_ID) or if it called it more than once. verify(mockAuthenticationService).isAuthenticated(USER_ID);</pre> A fake doesn’t use a mocking framework: it’s a lightweight implementation of an API that behaves like the real implementation, but isn't suitable for production (e.g. an in-memory database). Fakes can be used when you can't use a real implementation in your test (e.g. if the real implementation is too slow or it talks over the network). You shouldn't need to write your own fakes often since fakes should usually be created and maintained by the person or team that owns the real implementation. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// Creating the fake is fast and easy. AuthenticationService fakeAuthenticationService = new FakeAuthenticationService(); AccessManager accessManager = new AccessManager(fakeAuthenticationService); // The user shouldn't have access since the authentication service doesn't // know about the user. assertFalse(accessManager.userHasAccess(USER_ID)); // The user should have access after it's added to the authentication service. fakeAuthenticationService.addAuthenticatedUser(USER_ID); assertTrue(accessManager.userHasAccess(USER_ID));</pre> The term “test double” was coined by Gerard Meszaros in the book xUnit Test Patterns. You can find more information about test doubles in the book, or on the book’s <a href="http://xunitpatterns.com/Test%20Double%20Patterns.html">website</a>. You can also find a discussion about the different types of test doubles in this <a href="http://martinfowler.com/articles/mocksArentStubs.html">article</a> by Martin Fowler. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/file/d/0B_EQS8-2f7m5Y1lTcDRnekk3Rjg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. A test double is an object that can stand in for a real object in a test, similar to how a stunt double stands in for an actor in a movie. These are sometimes all commonly referred to as “mocks”, but it's important to distinguish between the different types of test doubles since they all have different uses. The most common types of test doubles are stubs, mocks, and fakes. A stub has no logic, and only returns what you tell it to return. Stubs can be used when you need an object to return specific values in order to get your code under test into a certain state. While it's usually easy to write stubs by hand, using a mocking framework is often a convenient way to reduce boilerplate. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// Pass in a stub that was created by a mocking framework. AccessManager accessManager = new AccessManager(stubAuthenticationService); // The user shouldn't have access when the authentication service returns false. when(stubAuthenticationService.isAuthenticated(USER_ID)).thenReturn(false); assertFalse(accessManager.userHasAccess(USER_ID)); // The user should have access when the authentication service returns true. when(stubAuthenticationService.isAuthenticated(USER_ID)).thenReturn(true); assertTrue(accessManager.userHasAccess(USER_ID)); </pre> A mock has expectations about the way it should be called, and a test should fail if it’s not called that way. Mocks are used to test interactions between objects, and are useful in cases where there are no other visible state changes or return results that you can verify (e.g. if your code reads from disk and you want to ensure that it doesn't do more than one disk read, you can use a mock to verify that the method that does the read is only called once). <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// Pass in a mock that was created by a mocking framework. AccessManager accessManager = new AccessManager(mockAuthenticationService); accessManager.userHasAccess(USER_ID); // The test should fail if accessManager.userHasAccess(USER_ID) didn't call // mockAuthenticationService.isAuthenticated(USER_ID) or if it called it more than once. verify(mockAuthenticationService).isAuthenticated(USER_ID);</pre> A fake doesn’t use a mocking framework: it’s a lightweight implementation of an API that behaves like the real implementation, but isn't suitable for production (e.g. an in-memory database). Fakes can be used when you can't use a real implementation in your test (e.g. if the real implementation is too slow or it talks over the network). You shouldn't need to write your own fakes often since fakes should usually be created and maintained by the person or team that owns the real implementation. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">// Creating the fake is fast and easy. AuthenticationService fakeAuthenticationService = new FakeAuthenticationService(); AccessManager accessManager = new AccessManager(fakeAuthenticationService); // The user shouldn't have access since the authentication service doesn't // know about the user. assertFalse(accessManager.userHasAccess(USER_ID)); // The user should have access after it's added to the authentication service. fakeAuthenticationService.addAuthenticatedUser(USER_ID); assertTrue(accessManager.userHasAccess(USER_ID));</pre> The term “test double” was coined by Gerard Meszaros in the book xUnit Test Patterns. You can find more information about test doubles in the book, or on the book’s <a href="http://xunitpatterns.com/Test%20Double%20Patterns.html">website</a>. You can also find a discussion about the different types of test doubles in this <a href="http://martinfowler.com/articles/mocksArentStubs.html">article</a> by Martin Fowler. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Know Your Test Doubles&url=https://testing.googleblog.com/2013/07/testing-on-toilet-know-your-test-doubles.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/07/testing-on-toilet-know-your-test-doubles.html#comments' style='font-weight: 500; text-decoration: underline;'>4 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/07/testing-on-toilet-know-your-test-doubles.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='3196266621721323261' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html' itemprop='url' title='Testing on the Toilet: Fake Your Way to Better Tests'> Testing on the Toilet: Fake Your Way to Better Tests </a> </h2> <div class='post-header'> <div class='published'> Friday, June 28, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Jonathan Rockway and Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/file/d/0B_EQS8-2f7m5MFdIN1NmOFFHZTg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. After many years of blogging, you decide to try out your blog platform's API. You start playing around with it, but then you realize: how can you tell that your code works without having your test talk to a remote blog server? <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void deletePostsWithTag(Tag tag) { for (Post post : blogService.getAllPosts()) { if (post.getTags().contains(tag)) { blogService.deletePost(post.getId()); } } } </pre> Fakes to the rescue! A fake is a lightweight implementation of an API that behaves like the real implementation, but isn't suitable for production. In the case of the blog service, all you care about is the ability to retrieve and delete posts. While a real blog service would need a database and support multiple frontend servers, you don’t need any of that to test your code, all you need is any implementation of the blog service API. You can achieve this with a simple in-memory implementation: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public class FakeBlogService implements BlogService { private final Set<Post> posts = new HashSet<Post>(); // Store posts in memory public void addPost(Post post) { posts.add(post); } public void deletePost(int id) { for (Post post : posts) { if (post.getId() == id) { posts.remove(post); return; } } throw new PostNotFoundException("No post with ID " + id); } public Set<Post> getAllPosts() { return posts; } } </pre> Now your tests can swap out the real blog service with the fake and the code under test won't even know the difference. Fakes are useful for when you can't use the real implementation in a test, such as if the real implementation is too slow (e.g. it takes several minutes to start up) or if it's non-deterministic (e.g. it talks to an external machine that may not be available when your test runs). You shouldn't need to write your own fakes often since each fake should be created and maintained by the person or team that owns the real implementation. If you’re using an API that doesn't provide a fake, it’s often easy to create one yourself: write a wrapper around the part of the code that you can't use in your tests, and create a fake for that wrapper. Remember to create the fake at the lowest level possible (e.g. if you can't use a database in your tests, fake out the database instead of faking out all of your classes that talk to the database), that way you'll have fewer fakes to maintain, and your tests will be executing more real code for important parts of your system. Fakes should have their own tests to ensure that they behave like the real implementation (e.g. if the real implementation throws an exception when given certain input, the fake implementation should also throw an exception when given the same input). One way to do this is to write tests against the API's public interface, and run those tests against both the real and fake implementations. If you still don't fully trust that your code will work in production if all your tests use a fake, you can write a small number of integration tests to ensure that your code will work with the real implementation. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Jonathan Rockway and Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="https://docs.google.com/file/d/0B_EQS8-2f7m5MFdIN1NmOFFHZTg/edit?usp=sharing">printer-friendly version</a> of this TotT episode and post it in your office. After many years of blogging, you decide to try out your blog platform's API. You start playing around with it, but then you realize: how can you tell that your code works without having your test talk to a remote blog server? <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void deletePostsWithTag(Tag tag) { for (Post post : blogService.getAllPosts()) { if (post.getTags().contains(tag)) { blogService.deletePost(post.getId()); } } } </pre> Fakes to the rescue! A fake is a lightweight implementation of an API that behaves like the real implementation, but isn't suitable for production. In the case of the blog service, all you care about is the ability to retrieve and delete posts. While a real blog service would need a database and support multiple frontend servers, you don’t need any of that to test your code, all you need is any implementation of the blog service API. You can achieve this with a simple in-memory implementation: <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public class FakeBlogService implements BlogService { private final Set<Post> posts = new HashSet<Post>(); // Store posts in memory public void addPost(Post post) { posts.add(post); } public void deletePost(int id) { for (Post post : posts) { if (post.getId() == id) { posts.remove(post); return; } } throw new PostNotFoundException("No post with ID " + id); } public Set<Post> getAllPosts() { return posts; } } </pre> Now your tests can swap out the real blog service with the fake and the code under test won't even know the difference. Fakes are useful for when you can't use the real implementation in a test, such as if the real implementation is too slow (e.g. it takes several minutes to start up) or if it's non-deterministic (e.g. it talks to an external machine that may not be available when your test runs). You shouldn't need to write your own fakes often since each fake should be created and maintained by the person or team that owns the real implementation. If you’re using an API that doesn't provide a fake, it’s often easy to create one yourself: write a wrapper around the part of the code that you can't use in your tests, and create a fake for that wrapper. Remember to create the fake at the lowest level possible (e.g. if you can't use a database in your tests, fake out the database instead of faking out all of your classes that talk to the database), that way you'll have fewer fakes to maintain, and your tests will be executing more real code for important parts of your system. Fakes should have their own tests to ensure that they behave like the real implementation (e.g. if the real implementation throws an exception when given certain input, the fake implementation should also throw an exception when given the same input). One way to do this is to write tests against the API's public interface, and run those tests against both the real and fake implementations. If you still don't fully trust that your code will work in production if all your tests use a fake, you can write a small number of integration tests to ensure that your code will work with the real implementation. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Fake Your Way to Better Tests&url=https://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html#comments' style='font-weight: 500; text-decoration: underline;'>2 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway' rel='tag'> Jonathan Rockway </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='1661480143086249368' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/06/optimal-logging.html' itemprop='url' title='Optimal Logging'> Optimal Logging </a> </h2> <div class='post-header'> <div class='published'> Friday, June 14, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> How long does it take to find the root cause of a failure in your system? Five minutes? Five days? If you answered close to five minutes, it’s very likely that your production system and tests have great logging. All too often, seemingly unessential features like logging, exception handling, and (dare I say it) testing are an implementation afterthought. Like exception handling and testing, you really need to have a strategy for logging in both your systems and your tests. Never underestimate the power of logging. With optimal logging, you can even eliminate the necessity for debuggers. Below are some guidelines that have been useful to me over the years. Channeling Goldilocks Never log too much. Massive, disk-quota burning logs are a clear indicator that little thought was put in to logging. If you log too much, you’ll need to devise complex approaches to minimize disk access, maintain log history, archive large quantities of data, and query these large sets of data. More importantly, you’ll make it very difficult to find valuable information in all the chatter. The only thing worse than logging too much is logging too little. There are normally two main goals of logging: help with bug investigation and event confirmation. If your log can’t explain the cause of a bug or whether a certain transaction took place, you are logging too little. Good things to log: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Important startup configuration</li> <li>Errors</li> <li>Warnings</li> <li>Changes to persistent data</li> <li>Requests and responses between major system components</li> <li>Significant state changes</li> <li>User interactions</li> <li>Calls with a known risk of failure</li> <li>Waits on conditions that could take measurable time to satisfy</li> <li>Periodic progress during long-running tasks</li> <li>Significant branch points of logic and conditions that led to the branch</li> <li>Summaries of processing steps or events from high level functions - Avoid logging every step of a complex process in low-level functions.</li> </ul> Bad things to log: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Function entry - Don’t log a function entry unless it is significant or logged at the debug level.</li> <li>Data within a loop - Avoid logging from many iterations of a loop. It is OK to log from iterations of small loops or to log periodically from large loops.</li> <li>Content of large messages or files - Truncate or summarize the data in some way that will be useful to debugging.</li> <li>Benign errors - Errors that are not really errors can confuse the log reader. This sometimes happens when exception handling is part of successful execution flow.</li> <li>Repetitive errors - Do not repetitively log the same or similar error. This can quickly fill a log and hide the actual cause. Frequency of error types is best handled by monitoring. Logs only need to capture detail for some of those errors.</li> </ul> There is More Than One Level Don't log everything at the same log level. Most logging libraries offer several log levels, and you can enable certain levels at system startup. This provides a convenient control for log verbosity. The classic levels are: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Debug - verbose and only useful while developing and/or debugging.</li> <li>Info - the most popular level.</li> <li>Warning - strange or unexpected states that are acceptable.</li> <li>Error - something went wrong, but the process can recover.</li> <li>Critical - the process cannot recover, and it will shutdown or restart.</li> </ul> Practically speaking, only two log configurations are needed: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Production - Every level is enabled except debug. If something goes wrong in production, the logs should reveal the cause.</li> <li>Development & Debug - While developing new code or trying to reproduce a production issue, enable all levels.</li> </ul> Test Logs Are Important Too Log quality is equally important in test and production code. When a test fails, the log should clearly show whether the failure was a problem with the test or production system. If it doesn't, then test logging is broken. Test logs should always contain: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Test execution environment</li> <li>Initial state</li> <li>Setup steps</li> <li>Test case steps</li> <li>Interactions with the system</li> <li>Expected results</li> <li>Actual results</li> <li>Teardown steps</li> </ul> Conditional Verbosity With Temporary Log Queues When errors occur, the log should contain a lot of detail. Unfortunately, detail that led to an error is often unavailable once the error is encountered. Also, if you’ve followed advice about not logging too much, your log records prior to the error record may not provide adequate detail. A good way to solve this problem is to create temporary, in-memory log queues. Throughout processing of a transaction, append verbose details about each step to the queue. If the transaction completes successfully, discard the queue and log a summary. If an error is encountered, log the content of the entire queue and the error. This technique is especially useful for test logging of system interactions. Failures and Flakiness Are Opportunities When production problems occur, you’ll obviously be focused on finding and correcting the problem, but you should also think about the logs. If you have a hard time determining the cause of an error, it's a great opportunity to improve your logging. Before fixing the problem, fix your logging so that the logs clearly show the cause. If this problem ever happens again, it’ll be much easier to identify. If you cannot reproduce the problem, or you have a flaky test, enhance the logs so that the problem can be tracked down when it happens again. Using failures to improve logging should be used throughout the development process. While writing new code, try to refrain from using debuggers and only use the logs. Do the logs describe what is going on? If not, the logging is insufficient. Might As Well Log Performance Data Logged timing data can help debug performance issues. For example, it can be very difficult to determine the cause of a timeout in a large system, unless you can trace the time spent on every significant processing step. This can be easily accomplished by logging the start and finish times of calls that can take measurable time: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Significant system calls</li> <li>Network requests</li> <li>CPU intensive operations</li> <li>Connected device interactions</li> <li>Transactions</li> </ul> Following the Trail Through Many Threads and Processes You should create unique identifiers for transactions that involve processing across many threads and/or processes. The initiator of the transaction should create the ID, and it should be passed to every component that performs work for the transaction. This ID should be logged by each component when logging information about the transaction. This makes it much easier to trace a specific transaction when many transactions are being processed concurrently. Monitoring and Logging Complement Each Other A production service should have both logging and monitoring. Monitoring provides a real-time statistical summary of the system state. It can alert you if a percentage of certain request types are failing, it is experiencing unusual traffic patterns, performance is degrading, or other anomalies occur. In some cases, this information alone will clue you to the cause of a problem. However, in most cases, a monitoring alert is simply a trigger for you to start an investigation. Monitoring shows the symptoms of problems. Logs provide details and state on individual transactions, so you can fully understand the cause of problems. <div> </div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> How long does it take to find the root cause of a failure in your system? Five minutes? Five days? If you answered close to five minutes, it’s very likely that your production system and tests have great logging. All too often, seemingly unessential features like logging, exception handling, and (dare I say it) testing are an implementation afterthought. Like exception handling and testing, you really need to have a strategy for logging in both your systems and your tests. Never underestimate the power of logging. With optimal logging, you can even eliminate the necessity for debuggers. Below are some guidelines that have been useful to me over the years. Channeling Goldilocks Never log too much. Massive, disk-quota burning logs are a clear indicator that little thought was put in to logging. If you log too much, you’ll need to devise complex approaches to minimize disk access, maintain log history, archive large quantities of data, and query these large sets of data. More importantly, you’ll make it very difficult to find valuable information in all the chatter. The only thing worse than logging too much is logging too little. There are normally two main goals of logging: help with bug investigation and event confirmation. If your log can’t explain the cause of a bug or whether a certain transaction took place, you are logging too little. Good things to log: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Important startup configuration</li> <li>Errors</li> <li>Warnings</li> <li>Changes to persistent data</li> <li>Requests and responses between major system components</li> <li>Significant state changes</li> <li>User interactions</li> <li>Calls with a known risk of failure</li> <li>Waits on conditions that could take measurable time to satisfy</li> <li>Periodic progress during long-running tasks</li> <li>Significant branch points of logic and conditions that led to the branch</li> <li>Summaries of processing steps or events from high level functions - Avoid logging every step of a complex process in low-level functions.</li> </ul> Bad things to log: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Function entry - Don’t log a function entry unless it is significant or logged at the debug level.</li> <li>Data within a loop - Avoid logging from many iterations of a loop. It is OK to log from iterations of small loops or to log periodically from large loops.</li> <li>Content of large messages or files - Truncate or summarize the data in some way that will be useful to debugging.</li> <li>Benign errors - Errors that are not really errors can confuse the log reader. This sometimes happens when exception handling is part of successful execution flow.</li> <li>Repetitive errors - Do not repetitively log the same or similar error. This can quickly fill a log and hide the actual cause. Frequency of error types is best handled by monitoring. Logs only need to capture detail for some of those errors.</li> </ul> There is More Than One Level Don't log everything at the same log level. Most logging libraries offer several log levels, and you can enable certain levels at system startup. This provides a convenient control for log verbosity. The classic levels are: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Debug - verbose and only useful while developing and/or debugging.</li> <li>Info - the most popular level.</li> <li>Warning - strange or unexpected states that are acceptable.</li> <li>Error - something went wrong, but the process can recover.</li> <li>Critical - the process cannot recover, and it will shutdown or restart.</li> </ul> Practically speaking, only two log configurations are needed: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Production - Every level is enabled except debug. If something goes wrong in production, the logs should reveal the cause.</li> <li>Development & Debug - While developing new code or trying to reproduce a production issue, enable all levels.</li> </ul> Test Logs Are Important Too Log quality is equally important in test and production code. When a test fails, the log should clearly show whether the failure was a problem with the test or production system. If it doesn't, then test logging is broken. Test logs should always contain: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Test execution environment</li> <li>Initial state</li> <li>Setup steps</li> <li>Test case steps</li> <li>Interactions with the system</li> <li>Expected results</li> <li>Actual results</li> <li>Teardown steps</li> </ul> Conditional Verbosity With Temporary Log Queues When errors occur, the log should contain a lot of detail. Unfortunately, detail that led to an error is often unavailable once the error is encountered. Also, if you’ve followed advice about not logging too much, your log records prior to the error record may not provide adequate detail. A good way to solve this problem is to create temporary, in-memory log queues. Throughout processing of a transaction, append verbose details about each step to the queue. If the transaction completes successfully, discard the queue and log a summary. If an error is encountered, log the content of the entire queue and the error. This technique is especially useful for test logging of system interactions. Failures and Flakiness Are Opportunities When production problems occur, you’ll obviously be focused on finding and correcting the problem, but you should also think about the logs. If you have a hard time determining the cause of an error, it's a great opportunity to improve your logging. Before fixing the problem, fix your logging so that the logs clearly show the cause. If this problem ever happens again, it’ll be much easier to identify. If you cannot reproduce the problem, or you have a flaky test, enhance the logs so that the problem can be tracked down when it happens again. Using failures to improve logging should be used throughout the development process. While writing new code, try to refrain from using debuggers and only use the logs. Do the logs describe what is going on? If not, the logging is insufficient. Might As Well Log Performance Data Logged timing data can help debug performance issues. For example, it can be very difficult to determine the cause of a timeout in a large system, unless you can trace the time spent on every significant processing step. This can be easily accomplished by logging the start and finish times of calls that can take measurable time: <ul style="line-height: 1em; margin-bottom: 0px; margin-top: 0px; padding-bottom: 0px; padding-top: 0px;"> <li>Significant system calls</li> <li>Network requests</li> <li>CPU intensive operations</li> <li>Connected device interactions</li> <li>Transactions</li> </ul> Following the Trail Through Many Threads and Processes You should create unique identifiers for transactions that involve processing across many threads and/or processes. The initiator of the transaction should create the ID, and it should be passed to every component that performs work for the transaction. This ID should be logged by each component when logging information about the transaction. This makes it much easier to trace a specific transaction when many transactions are being processed concurrently. Monitoring and Logging Complement Each Other A production service should have both logging and monitoring. Monitoring provides a real-time statistical summary of the system state. It can alert you if a percentage of certain request types are failing, it is experiencing unusual traffic patterns, performance is degrading, or other anomalies occur. In some cases, this information alone will clue you to the cause of a problem. However, in most cases, a monitoring alert is simply a trigger for you to start an investigation. Monitoring shows the symptoms of problems. Logs provide details and state on individual transactions, so you can fully understand the cause of problems. <div> </div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/06/optimal-logging.html#comments' style='font-weight: 500; text-decoration: underline;'>10 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/06/optimal-logging.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> </div> </div> </div> <div class='post' data-id='532776740027781314' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html' itemprop='url' title='Testing on the Toilet: Don’t Overuse Mocks'> Testing on the Toilet: Don’t Overuse Mocks </a> </h2> <div class='post-header'> <div class='published'> Tuesday, May 28, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="//goo.gl/IEfP0">printer-friendly version</a> of this TotT episode and post it in your office. When writing tests for your code, it can seem easy to ignore your code's dependencies by mocking them out. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void testCreditCardIsCharged() { paymentProcessor = new PaymentProcessor(mockCreditCardServer); when(mockCreditCardServer.isServerAvailable()).thenReturn(true); when(mockCreditCardServer.beginTransaction()).thenReturn(mockTransactionManager); when(mockTransactionManager.getTransaction()).thenReturn(transaction); when(mockCreditCardServer.pay(transaction, creditCard, 500)).thenReturn(mockPayment); when(mockPayment.isOverMaxBalance()).thenReturn(false); paymentProcessor.processPayment(creditCard, Money.dollars(500)); verify(mockCreditCardServer).pay(transaction, creditCard, 500); }</pre> However, not using mocks can sometimes result in tests that are simpler and more useful. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void testCreditCardIsCharged() { paymentProcessor = new PaymentProcessor(creditCardServer); paymentProcessor.processPayment(creditCard, Money.dollars(500)); assertEquals(500, creditCardServer.getMostRecentCharge(creditCard)); }</pre> Overusing mocks can cause several problems: - Tests can be harder to understand. Instead of just a straightforward usage of your code (e.g. pass in some values to the method under test and check the return result), you need to include extra code to tell the mocks how to behave. Having this extra code detracts from the actual intent of what you’re trying to test, and very often this code is hard to understand if you're not familiar with the implementation of the production code. - Tests can be harder to maintain. When you tell a mock how to behave, you're leaking implementation details of your code into your test. When implementation details in your production code change, you'll need to update your tests to reflect these changes. Tests should typically know little about the code's implementation, and should focus on testing the code's public interface. - Tests can provide less assurance that your code is working properly. When you tell a mock how to behave, the only assurance you get with your tests is that your code will work if your mocks behave exactly like your real implementations. This can be very hard to guarantee, and the problem gets worse as your code changes over time, as the behavior of the real implementations is likely to get out of sync with your mocks. Some signs that you're overusing mocks are if you're mocking out more than one or two classes, or if one of your mocks specifies how more than one or two methods should behave. If you're trying to read a test that uses mocks and find yourself mentally stepping through the code being tested in order to understand the test, then you're probably overusing mocks. Sometimes you can't use a real dependency in a test (e.g. if it's too slow or talks over the network), but there may better options than using mocks, such as a hermetic local server (e.g. a credit card server that you start up on your machine specifically for the test) or a fake implementation (e.g. an in-memory credit card server). For more information about using hermetic servers, see <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">http://googletesting.blogspot.com/2012/10/hermetic-servers.html</a>. Stay tuned for a future Testing on the Toilet episode about using fake implementations. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a <a href="//goo.gl/IEfP0">printer-friendly version</a> of this TotT episode and post it in your office. When writing tests for your code, it can seem easy to ignore your code's dependencies by mocking them out. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void testCreditCardIsCharged() { paymentProcessor = new PaymentProcessor(mockCreditCardServer); when(mockCreditCardServer.isServerAvailable()).thenReturn(true); when(mockCreditCardServer.beginTransaction()).thenReturn(mockTransactionManager); when(mockTransactionManager.getTransaction()).thenReturn(transaction); when(mockCreditCardServer.pay(transaction, creditCard, 500)).thenReturn(mockPayment); when(mockPayment.isOverMaxBalance()).thenReturn(false); paymentProcessor.processPayment(creditCard, Money.dollars(500)); verify(mockCreditCardServer).pay(transaction, creditCard, 500); }</pre> However, not using mocks can sometimes result in tests that are simpler and more useful. <pre style="background: #f9f9f9; border-color: #7b7b7b; border-style: solid; border-width: 1px; color: black; overflow: auto; padding: 10px;">public void testCreditCardIsCharged() { paymentProcessor = new PaymentProcessor(creditCardServer); paymentProcessor.processPayment(creditCard, Money.dollars(500)); assertEquals(500, creditCardServer.getMostRecentCharge(creditCard)); }</pre> Overusing mocks can cause several problems: - Tests can be harder to understand. Instead of just a straightforward usage of your code (e.g. pass in some values to the method under test and check the return result), you need to include extra code to tell the mocks how to behave. Having this extra code detracts from the actual intent of what you’re trying to test, and very often this code is hard to understand if you're not familiar with the implementation of the production code. - Tests can be harder to maintain. When you tell a mock how to behave, you're leaking implementation details of your code into your test. When implementation details in your production code change, you'll need to update your tests to reflect these changes. Tests should typically know little about the code's implementation, and should focus on testing the code's public interface. - Tests can provide less assurance that your code is working properly. When you tell a mock how to behave, the only assurance you get with your tests is that your code will work if your mocks behave exactly like your real implementations. This can be very hard to guarantee, and the problem gets worse as your code changes over time, as the behavior of the real implementations is likely to get out of sync with your mocks. Some signs that you're overusing mocks are if you're mocking out more than one or two classes, or if one of your mocks specifies how more than one or two methods should behave. If you're trying to read a test that uses mocks and find yourself mentally stepping through the code being tested in order to understand the test, then you're probably overusing mocks. Sometimes you can't use a real dependency in a test (e.g. if it's too slow or talks over the network), but there may better options than using mocks, such as a hermetic local server (e.g. a credit card server that you start up on your machine specifically for the test) or a fake implementation (e.g. an in-memory credit card server). For more information about using hermetic servers, see <a href="http://googletesting.blogspot.com/2012/10/hermetic-servers.html">http://googletesting.blogspot.com/2012/10/hermetic-servers.html</a>. Stay tuned for a future Testing on the Toilet episode about using fake implementations. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Don’t Overuse Mocks&url=https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html#comments' style='font-weight: 500; text-decoration: underline;'>21 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='8981809450625382685' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/05/gtac-2013-wrap-up.html' itemprop='url' title='GTAC 2013 Wrap-up'> GTAC 2013 Wrap-up </a> </h2> <div class='post-header'> <div class='published'> Saturday, May 04, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee <div style="clear: both;"> </div> <img border="0" height="180" src="https://developers.google.com/google-test-automation-conference/2013/images/event_thumb/003.jpg" style="float: left; margin: 0px 10px 10px 0px;" /> The Google Test Automation Conference (<a href="https://developers.google.com/google-test-automation-conference/">GTAC</a>) was held last week in NYC on April 23rd & 24th. The theme for this year's conference was focused on Mobile and Media. We were fortunate to have a cross section of attendees and presenters from industry and academia. This year’s talks focused on trends we are seeing in industry combined with compelling talks on tools and infrastructure that can have a direct impact on our products. We believe we achieved a conference that was focused for engineers by engineers. GTAC 2013 demonstrated that there is a strong trend toward the emergence of test engineering as a computer science discipline across companies and academia alike. <div style="clear: both;"> </div> <img border="0" height="180" src="https://developers.google.com/google-test-automation-conference/2013/images/event_thumb/044.jpg" style="float: right; margin: 0px 0px 10px 10px;" /> All of the <a href="https://developers.google.com/google-test-automation-conference/2013/presentations">slides, video recordings</a>, and <a href="https://developers.google.com/google-test-automation-conference/2013/photos">photos</a> are now available on the GTAC site. Thank you to all the <a href="https://developers.google.com/google-test-automation-conference/2013/speakers">speakers</a> and attendees who made this event spectacular. We are already looking forward to the next GTAC. If you have suggestions for next year’s location or theme, please comment on this post. To receive GTAC updates, subscribe to the <a href="http://googletesting.blogspot.com/">Google Testing Blog</a>. <div style="clear: both;"> </div> <img border="0" height="180" src="https://developers.google.com/google-test-automation-conference/2013/images/event_thumb/065.jpg" style="float: left; margin: 0px 10px 10px 0px;" /> Here are some responses to GTAC 2013: “My first GTAC, and one of the best conferences of any kind I've ever been to. The talks were consistently great and the chance to interact with so many experts from all over the map was priceless.” - Gareth Bowles, <a href="http://www.netflix.com/">Netflix</a> “Adding my own thanks as a speaker (and consumer of the material, I learned a lot from the other speakers) -- this was amazingly well run, and had facilities that I've seen many larger conferences not provide. I got everything I wanted from attending and more!” - James Waldrop, <a href="https://twitter.com/">Twitter</a> “This was a wonderful conference. I learned so much in two days and met some great people. Can't wait to get back to Denver and use all this newly acquired knowledge!” - Crystal Preston-Watson, <a href="https://www.pingidentity.com/">Ping Identity</a> “GTAC is hands down the smoothest conference/event I've attended. Well done to Google and all involved.” - Alister Scott, <a href="http://www.thoughtworks.com/">ThoughtWorks</a> “Thanks and compliments for an amazingly brain activity spurring event. I returned very inspired. First day back at work and the first thing I am doing is looking into improving our build automation and speed (1 min is too long. We are not building that much, groovy is dynamic).” - Irina Muchnik, <a href="http://www.zynxhealth.com/">Zynx Health</a> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee <div style="clear: both;"> </div> <img border="0" height="180" src="https://developers.google.com/google-test-automation-conference/2013/images/event_thumb/003.jpg" style="float: left; margin: 0px 10px 10px 0px;" /> The Google Test Automation Conference (<a href="https://developers.google.com/google-test-automation-conference/">GTAC</a>) was held last week in NYC on April 23rd & 24th. The theme for this year's conference was focused on Mobile and Media. We were fortunate to have a cross section of attendees and presenters from industry and academia. This year’s talks focused on trends we are seeing in industry combined with compelling talks on tools and infrastructure that can have a direct impact on our products. We believe we achieved a conference that was focused for engineers by engineers. GTAC 2013 demonstrated that there is a strong trend toward the emergence of test engineering as a computer science discipline across companies and academia alike. <div style="clear: both;"> </div> <img border="0" height="180" src="https://developers.google.com/google-test-automation-conference/2013/images/event_thumb/044.jpg" style="float: right; margin: 0px 0px 10px 10px;" /> All of the <a href="https://developers.google.com/google-test-automation-conference/2013/presentations">slides, video recordings</a>, and <a href="https://developers.google.com/google-test-automation-conference/2013/photos">photos</a> are now available on the GTAC site. Thank you to all the <a href="https://developers.google.com/google-test-automation-conference/2013/speakers">speakers</a> and attendees who made this event spectacular. We are already looking forward to the next GTAC. If you have suggestions for next year’s location or theme, please comment on this post. To receive GTAC updates, subscribe to the <a href="http://googletesting.blogspot.com/">Google Testing Blog</a>. <div style="clear: both;"> </div> <img border="0" height="180" src="https://developers.google.com/google-test-automation-conference/2013/images/event_thumb/065.jpg" style="float: left; margin: 0px 10px 10px 0px;" /> Here are some responses to GTAC 2013: “My first GTAC, and one of the best conferences of any kind I've ever been to. The talks were consistently great and the chance to interact with so many experts from all over the map was priceless.” - Gareth Bowles, <a href="http://www.netflix.com/">Netflix</a> “Adding my own thanks as a speaker (and consumer of the material, I learned a lot from the other speakers) -- this was amazingly well run, and had facilities that I've seen many larger conferences not provide. I got everything I wanted from attending and more!” - James Waldrop, <a href="https://twitter.com/">Twitter</a> “This was a wonderful conference. I learned so much in two days and met some great people. Can't wait to get back to Denver and use all this newly acquired knowledge!” - Crystal Preston-Watson, <a href="https://www.pingidentity.com/">Ping Identity</a> “GTAC is hands down the smoothest conference/event I've attended. Well done to Google and all involved.” - Alister Scott, <a href="http://www.thoughtworks.com/">ThoughtWorks</a> “Thanks and compliments for an amazingly brain activity spurring event. I returned very inspired. First day back at work and the first thing I am doing is looking into improving our build automation and speed (1 min is too long. We are not building that much, groovy is dynamic).” - Irina Muchnik, <a href="http://www.zynxhealth.com/">Zynx Health</a> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC 2013 Wrap-up&url=https://testing.googleblog.com/2013/05/gtac-2013-wrap-up.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/05/gtac-2013-wrap-up.html#comments' style='font-weight: 500; text-decoration: underline;'>8 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/05/gtac-2013-wrap-up.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='5079707763558922357' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/04/gtac-is-almost-here.html' itemprop='url' title='GTAC is Almost Here!'> GTAC is Almost Here! </a> </h2> <div class='post-header'> <div class='published'> Saturday, April 13, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee GTAC is just around the corner, and we’re all very busy and excited. I know we say this every year, but this is going to be the best GTAC ever! We have updated the GTAC site with important details: <ul> <li style="line-height: 1em;"><a href="https://developers.google.com/google-test-automation-conference/2013/schedule">The Schedule</a> </li> <li style="line-height: 1em;"><a href="https://developers.google.com/google-test-automation-conference/2013/speakers">Speaker Profiles</a> </li> <li style="line-height: 1em;"><a href="https://developers.google.com/google-test-automation-conference/2013/committee">Committee Profiles</a> </li> </ul> If you are on the attendance list, we’ll see you on April 23rd. If not, check out the <a href="https://developers.google.com/google-test-automation-conference/2013/stream">Live Stream</a> page where you can watch the conference live and can get involved in Q&A after each talk. Perhaps your team can gather in a conference room and attend remotely. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee GTAC is just around the corner, and we’re all very busy and excited. I know we say this every year, but this is going to be the best GTAC ever! We have updated the GTAC site with important details: <ul> <li style="line-height: 1em;"><a href="https://developers.google.com/google-test-automation-conference/2013/schedule">The Schedule</a> </li> <li style="line-height: 1em;"><a href="https://developers.google.com/google-test-automation-conference/2013/speakers">Speaker Profiles</a> </li> <li style="line-height: 1em;"><a href="https://developers.google.com/google-test-automation-conference/2013/committee">Committee Profiles</a> </li> </ul> If you are on the attendance list, we’ll see you on April 23rd. If not, check out the <a href="https://developers.google.com/google-test-automation-conference/2013/stream">Live Stream</a> page where you can watch the conference live and can get involved in Q&A after each talk. Perhaps your team can gather in a conference room and attend remotely. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:GTAC is Almost Here!&url=https://testing.googleblog.com/2013/04/gtac-is-almost-here.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/04/gtac-is-almost-here.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/04/gtac-is-almost-here.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='2012075819086436887' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/04/two-new-videos-about-testing-at-google.html' itemprop='url' title='Two New Videos About Testing at Google'> Two New Videos About Testing at Google </a> </h2> <div class='post-header'> <div class='published'> Friday, April 12, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> We have two excellent, new videos to share about testing at Google. If you are curious about the work that our Test Engineers (TEs) and Software Engineers in Test (SETs) do, you’ll find both of these videos very interesting. The <a href="//www.youtube.com/lifeatgoogle">Life at Google</a> team produced a video series called <a href="//www.youtube.com/watch?v=eZHEnPCEe0o&feature=c4-overview-vl&list=PLllx_3tLoo4fd1deqnzvyZrIrJzRdSC6-">Do Cool Things That Matter</a>. This series includes a video from an SET and TE on the Maps team (Sean Jordan and Yvette Nameth) discussing their work on the Google Maps team. <a href="//www.youtube.com/watch?v=FcwMMEsRFW4&list=PLllx_3tLoo4fd1deqnzvyZrIrJzRdSC6-">Meet Yvette and Sean from the Google Maps Test Team</a> <div style="text-align: center;"> <iframe allowfullscreen="" frameborder="0" height="315" src="//www.youtube.com/embed/FcwMMEsRFW4?list=PLllx_3tLoo4fd1deqnzvyZrIrJzRdSC6-" width="560"></iframe> </div> The <a href="https://plus.sandbox.google.com/u/0/101571483150813305324">Google Students</a> team hosted a <a href="//www.google.com/+/learnmore/hangouts/onair.html">Hangouts On Air</a> event with several Google SETs (Diego Salas, Karin Lundberg, Jonathan Velasquez, Chaitali Narla, and Dave Chen) discussing the SET role. <a href="https://plus.google.com/u/0/101571483150813305324/posts/diXcYNHmMvM">Software Engineers in Test at Google - Covering your (Code)Bases</a> <div style="text-align: center;"> <iframe allowfullscreen="" frameborder="0" height="315" src="//www.youtube.com/embed/OZk8nlcpe3w" width="560"></iframe> </div> Interested in joining the ranks of TEs or SETs at Google? Search for <a href="//goo.gl/I0w5J">Google test jobs</a>. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> We have two excellent, new videos to share about testing at Google. If you are curious about the work that our Test Engineers (TEs) and Software Engineers in Test (SETs) do, you’ll find both of these videos very interesting. The <a href="//www.youtube.com/lifeatgoogle">Life at Google</a> team produced a video series called <a href="//www.youtube.com/watch?v=eZHEnPCEe0o&feature=c4-overview-vl&list=PLllx_3tLoo4fd1deqnzvyZrIrJzRdSC6-">Do Cool Things That Matter</a>. This series includes a video from an SET and TE on the Maps team (Sean Jordan and Yvette Nameth) discussing their work on the Google Maps team. <a href="//www.youtube.com/watch?v=FcwMMEsRFW4&list=PLllx_3tLoo4fd1deqnzvyZrIrJzRdSC6-">Meet Yvette and Sean from the Google Maps Test Team</a> <div style="text-align: center;"> <iframe allowfullscreen="" frameborder="0" height="315" src="//www.youtube.com/embed/FcwMMEsRFW4?list=PLllx_3tLoo4fd1deqnzvyZrIrJzRdSC6-" width="560"></iframe> </div> The <a href="https://plus.sandbox.google.com/u/0/101571483150813305324">Google Students</a> team hosted a <a href="//www.google.com/+/learnmore/hangouts/onair.html">Hangouts On Air</a> event with several Google SETs (Diego Salas, Karin Lundberg, Jonathan Velasquez, Chaitali Narla, and Dave Chen) discussing the SET role. <a href="https://plus.google.com/u/0/101571483150813305324/posts/diXcYNHmMvM">Software Engineers in Test at Google - Covering your (Code)Bases</a> <div style="text-align: center;"> <iframe allowfullscreen="" frameborder="0" height="315" src="//www.youtube.com/embed/OZk8nlcpe3w" width="560"></iframe> </div> Interested in joining the ranks of TEs or SETs at Google? Search for <a href="//goo.gl/I0w5J">Google test jobs</a>. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Two New Videos About Testing at Google&url=https://testing.googleblog.com/2013/04/two-new-videos-about-testing-at-google.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/04/two-new-videos-about-testing-at-google.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/04/two-new-videos-about-testing-at-google.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/Chaitali%20Narla' rel='tag'> Chaitali Narla </a> , <a class='label' href='https://testing.googleblog.com/search/label/Dave%20Chen' rel='tag'> Dave Chen </a> , <a class='label' href='https://testing.googleblog.com/search/label/Diego%20Salas' rel='tag'> Diego Salas </a> , <a class='label' href='https://testing.googleblog.com/search/label/Jobs' rel='tag'> Jobs </a> , <a class='label' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez' rel='tag'> Jonathan Velasquez </a> , <a class='label' href='https://testing.googleblog.com/search/label/Karin%20Lundberg' rel='tag'> Karin Lundberg </a> , <a class='label' href='https://testing.googleblog.com/search/label/Sean%20Jordan' rel='tag'> Sean Jordan </a> , <a class='label' href='https://testing.googleblog.com/search/label/Yvette%20Nameth' rel='tag'> Yvette Nameth </a> </div> </div> </div> <div class='post' data-id='8874631609951045109' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/03/testing-on-toilet-testing-state-vs.html' itemprop='url' title='Testing on the Toilet: Testing State vs. Testing Interactions'> Testing on the Toilet: Testing State vs. Testing Interactions </a> </h2> <div class='post-header'> <div class='published'> Friday, March 22, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a printer-friendly <a href="//goo.gl/8x1Ha">version</a> of this TotT episode and post it in your office. There are typically two ways a unit test can verify that the code under test is working properly: by testing state or by testing interactions. What’s the difference between these? Testing state means you're verifying that the code under test returns the right results. <pre style="background: #fffcd0; border: solid 1px #AAAAAA; overflow: auto; padding: 10px;">public void testSortNumbers() { NumberSorter numberSorter = new NumberSorter(quicksort, bubbleSort); // Verify that the returned list is sorted. It doesn't matter which sorting // algorithm is used, as long as the right result is returned. assertEquals( new ArrayList(1, 2, 3), numberSorter.sortNumbers(new ArrayList(3, 1, 2))); }</pre> Testing interactions means you're verifying that the code under test calls certain methods properly. <pre style="background: #fffcd0; border: solid 1px #AAAAAA; overflow: auto; padding: 10px;">public void testSortNumbers_quicksortIsUsed() { // Pass in mocks to the class and call the method under test. NumberSorter numberSorter = new NumberSorter(mockQuicksort, mockBubbleSort); numberSorter.sortNumbers(new ArrayList(3, 1, 2)); // Verify that numberSorter.sortNumbers() used quicksort. The test should // fail if mockQuicksort.sort() is never called or if it's called with the // wrong arguments (e.g. if mockBubbleSort is used to sort the numbers). verify(mockQuicksort).sort(new ArrayList(3, 1, 2)); }</pre> The second test may result in good code coverage, but it doesn't tell you whether sorting works properly, only that quicksort.sort() was called. Just because a test that uses interactions is passing doesn't mean the code is working properly. This is why in most cases, you want to test state, not interactions. In general, interactions should be tested when correctness doesn't just depend on what the code's output is, but also how the output is determined. In the above example, you would only want to test interactions in addition to testing state if it's important that quicksort is used (e.g. the method would run too slowly with a different sorting algorithm), otherwise the test using interactions is unnecessary. What are some other examples of cases where you want to test interactions? - The code under test calls a method where differences in the number or order of calls would cause undesired behavior, such as side effects (e.g. you only want one email to be sent), latency (e.g. you only want a certain number of disk reads to occur) or multithreading issues (e.g. your code will deadlock if it calls some methods in the wrong order). Testing interactions ensures that your tests will fail if these methods aren't called properly. - You're testing a UI where the rendering details of the UI are abstracted away from the UI logic (e.g. using MVC or MVP). In tests for your controller/presenter, you only care that a certain method of the view was called, not what was actually rendered, so you can test interactions with the view. Similarly, when testing the view, you can test interactions with the controller/presenter. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> By Andrew Trenk This article was adapted from a <a href="http://googletesting.blogspot.com/2007/01/introducing-testing-on-toilet.html">Google Testing on the Toilet</a> (TotT) episode. You can download a printer-friendly <a href="//goo.gl/8x1Ha">version</a> of this TotT episode and post it in your office. There are typically two ways a unit test can verify that the code under test is working properly: by testing state or by testing interactions. What’s the difference between these? Testing state means you're verifying that the code under test returns the right results. <pre style="background: #fffcd0; border: solid 1px #AAAAAA; overflow: auto; padding: 10px;">public void testSortNumbers() { NumberSorter numberSorter = new NumberSorter(quicksort, bubbleSort); // Verify that the returned list is sorted. It doesn't matter which sorting // algorithm is used, as long as the right result is returned. assertEquals( new ArrayList(1, 2, 3), numberSorter.sortNumbers(new ArrayList(3, 1, 2))); }</pre> Testing interactions means you're verifying that the code under test calls certain methods properly. <pre style="background: #fffcd0; border: solid 1px #AAAAAA; overflow: auto; padding: 10px;">public void testSortNumbers_quicksortIsUsed() { // Pass in mocks to the class and call the method under test. NumberSorter numberSorter = new NumberSorter(mockQuicksort, mockBubbleSort); numberSorter.sortNumbers(new ArrayList(3, 1, 2)); // Verify that numberSorter.sortNumbers() used quicksort. The test should // fail if mockQuicksort.sort() is never called or if it's called with the // wrong arguments (e.g. if mockBubbleSort is used to sort the numbers). verify(mockQuicksort).sort(new ArrayList(3, 1, 2)); }</pre> The second test may result in good code coverage, but it doesn't tell you whether sorting works properly, only that quicksort.sort() was called. Just because a test that uses interactions is passing doesn't mean the code is working properly. This is why in most cases, you want to test state, not interactions. In general, interactions should be tested when correctness doesn't just depend on what the code's output is, but also how the output is determined. In the above example, you would only want to test interactions in addition to testing state if it's important that quicksort is used (e.g. the method would run too slowly with a different sorting algorithm), otherwise the test using interactions is unnecessary. What are some other examples of cases where you want to test interactions? - The code under test calls a method where differences in the number or order of calls would cause undesired behavior, such as side effects (e.g. you only want one email to be sent), latency (e.g. you only want a certain number of disk reads to occur) or multithreading issues (e.g. your code will deadlock if it calls some methods in the wrong order). Testing interactions ensures that your tests will fail if these methods aren't called properly. - You're testing a UI where the rendering details of the UI are abstracted away from the UI logic (e.g. using MVC or MVP). In tests for your controller/presenter, you only care that a certain method of the view was called, not what was actually rendered, so you can test interactions with the view. Similarly, when testing the view, you can test interactions with the controller/presenter. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Testing on the Toilet: Testing State vs. Testing Interactions&url=https://testing.googleblog.com/2013/03/testing-on-toilet-testing-state-vs.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/03/testing-on-toilet-testing-state-vs.html#comments' style='font-weight: 500; text-decoration: underline;'>6 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/03/testing-on-toilet-testing-state-vs.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Andrew%20Trenk' rel='tag'> Andrew Trenk </a> , <a class='label' href='https://testing.googleblog.com/search/label/TotT' rel='tag'> TotT </a> </div> </div> </div> <div class='post' data-id='870791818819003209' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/03/announcing-gtac-2013-agenda.html' itemprop='url' title='Announcing the GTAC 2013 Agenda'> Announcing the GTAC 2013 Agenda </a> </h2> <div class='post-header'> <div class='published'> Wednesday, March 06, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee We have completed selection and confirmation of all speakers for GTAC 2013. You can find the detailed agenda at:   <a href="http://developers.google.com/gtac/2013/schedule">developers.google.com/gtac/2013/schedule</a> Thank you to all who submitted proposals! It was very hard to make selections from so many fantastic submissions. If you were not extended an invitation, don’t forget that you can join us via YouTube live streaming. We’ll be setting up <a href="//www.google.com/moderator/">Google Moderator</a>, so remote attendees can get involved in Q&A after each talk. Information about live streaming, Moderator, and other details will be posted on the GTAC site soon and announced here. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> by <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee We have completed selection and confirmation of all speakers for GTAC 2013. You can find the detailed agenda at:   <a href="http://developers.google.com/gtac/2013/schedule">developers.google.com/gtac/2013/schedule</a> Thank you to all who submitted proposals! It was very hard to make selections from so many fantastic submissions. If you were not extended an invitation, don’t forget that you can join us via YouTube live streaming. We’ll be setting up <a href="//www.google.com/moderator/">Google Moderator</a>, so remote attendees can get involved in Q&A after each talk. Information about live streaming, Moderator, and other details will be posted on the GTAC site soon and announced here. <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Announcing the GTAC 2013 Agenda&url=https://testing.googleblog.com/2013/03/announcing-gtac-2013-agenda.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/03/announcing-gtac-2013-agenda.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/03/announcing-gtac-2013-agenda.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='5497299513095045520' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/01/the-deadline-to-signup-for-gtac-is-jan.html' itemprop='url' title='The Deadline to Signup for GTAC is Jan 23'> The Deadline to Signup for GTAC is Jan 23 </a> </h2> <div class='post-header'> <div class='published'> Friday, January 18, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <div dir="ltr" style="text-align: left;" trbidi="on"> By <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee If you would like to attend or speak at GTAC 2013, the deadline to sign-up is January 23rd, 2013. We are really excited about hosting this event at our fabulous New York City office. We’ve received many interesting presentation proposals so far, and this year’s GTAC will certainly be a fascinating, important event for test automation professionals. We are still accepting proposals, so it’s not too late to add yours for consideration. You can find details about the conference at our new site:   <a href="http://developers.google.com/gtac">developers.google.com/gtac</a> We will be making regular updates to this site over the next several weeks. For those that have already signed up to attend or speak, we will contact you directly in early February.</div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> <div dir="ltr" style="text-align: left;" trbidi="on"> By <a href="http://anthonyvallone.com/">Anthony Vallone</a> on behalf of the GTAC Committee If you would like to attend or speak at GTAC 2013, the deadline to sign-up is January 23rd, 2013. We are really excited about hosting this event at our fabulous New York City office. We’ve received many interesting presentation proposals so far, and this year’s GTAC will certainly be a fascinating, important event for test automation professionals. We are still accepting proposals, so it’s not too late to add yours for consideration. You can find details about the conference at our new site:   <a href="http://developers.google.com/gtac">developers.google.com/gtac</a> We will be making regular updates to this site over the next several weeks. For those that have already signed up to attend or speak, we will contact you directly in early February.</div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:The Deadline to Signup for GTAC is Jan 23&url=https://testing.googleblog.com/2013/01/the-deadline-to-signup-for-gtac-is-jan.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/01/the-deadline-to-signup-for-gtac-is-jan.html#comments' style='font-weight: 500; text-decoration: underline;'>No comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/01/the-deadline-to-signup-for-gtac-is-jan.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Anthony%20Vallone' rel='tag'> Anthony Vallone </a> , <a class='label' href='https://testing.googleblog.com/search/label/GTAC' rel='tag'> GTAC </a> </div> </div> </div> <div class='post' data-id='4993778204490136867' itemscope='' itemtype='http://schema.org/BlogPosting'> <h2 class='title' itemprop='name'> <a href='https://testing.googleblog.com/2013/01/test-engineers-google.html' itemprop='url' title='Test Engineers @ Google'> Test Engineers @ Google </a> </h2> <div class='post-header'> <div class='published'> Monday, January 14, 2013 </div> </div> <div class='post-body'> <div class='post-content' itemprop='articleBody'> <script type='text/template'> <div dir="ltr" style="text-align: left;" trbidi="on"> By Yvette Nameth At Google, we’re very big into highlighting individuals’ strengths and using them to make teams and products better. However, we frequently get asked “What do Test Engineers (aka TEs) do?” I pause when I get this question since it’s hard to speak for my peers - I test <a href="http://google-latlong.blogspot.com/2011/07/evolving-look-of-google-maps-redux.html">Google Maps rendering</a>, which is just one small portion of what Google’s Test Engineers test. In order to get a clearer picture of what Test Engineers are responsible for, I chatted with three of my colleagues. We were able to identify the underlying Test Engineers’ similarities, while highlighting the differences. So what common themes do Test Engineers specialize in at Google? <div style="margin-left: 20px;"> We’re product experts: Test Engineers need to become a “go-to” person for how their product works and integrates with other Google products. (You aren’t expected to have this before working with a product, but you need to figure out how to become one on any product you work on!) TEs need to understand use cases and contracts with other services, products, and features. We aren’t expected to write unit tests for other engineer’s code; instead we ensure product quality on the functional and integration aspects of the product. We’re flexible: Test Engineers are required to switch tasks and re-prioritize frequently. From unplanned catastrophes, to shifting launch calendars, to people asking us questions, our work is filled with interrupts. We determine how to ensure quality in the face of the interrupts. We also modify our tests based on the pace of the development and understand that there is no one right way to test a product. Test Engineers adapt tools to meet their needs and understand when a tool just can’t get the job done. We’re clear communicators: We have to be able to communicate via test plans, design docs, bugs, email and code. Every day we work with a wide variety of people in different roles: Software Engineers, Software Engineers in Test, Product Managers, Usability Researchers, Designers, Legal Counsel, etc. We need to address these different audiences to make sure we’re either gathering the information that will help us build better strategies or presenting feedback that will help influence the product. We’re good at coordination: We are people who use our “in between the product and user” status to coordinate integration testing efforts between products. We may coordinate manual testing efforts by our manual testers; or we may make sure that test gaps are being addressed by “someone” (Test Engineer, Software Engineer in Test, or Software Engineer). We put our product knowledge and communication together with a bit of coordination and make sure that bugs are looked at and the product is getting tested hourly / daily. We have impact: Google Test Engineers have big impact. We hold responsibility thinking of ways that our products could fail in “real scenarios”; and then we add tests to make sure that the worst won’t come to pass. How big is this? Well, in my case, I’m responsible for making sure Google Maps represents a map that is useful to my relatives in the middle of rural Montana as well as my friends living in London, Paris or Sydney. When you add to that the billions of other users in different regions, speaking different languages and using the map for different reasons, I know that my testing is impacting their ability to get around and find out information about the physical world around them safely. We code: The other most common question is “Do you write code?” The answer is yes; Test Engineers at Google do code. </div> The three aspects that generally differentiate what a Test Engineer does day-to-day depend on the following: <div style="margin-left: 20px;"> Individual’s Strengths & Interests: Everyone is different and every TE has different passions, strengths and areas of expertise. Thankfully, Google’s a big enough company that many different areas of testing are available, and we gravitate to testing products we like. All TEs start with core competencies in testing, coding, and algorithms. How a TE applies this knowledge varies. The Type of Product: Desktop, web app or mobile? Frontend or backend? The technologies that our products use and run on create a lot of variation in what and how we test. The Product’s History / Lifecycle: Early concept products don’t resemble those that exist in production. And the amount of testing that a product already has will determine what testing the TE is focused on. We work creating a test roadmap that parallels the product’s development cycle and addresses any testing gaps. </div> If you still want to know what the day in the life of a Test Engineer entails, we’ll never be able to give you a general answer for that. Instead I suggest that you check out <a href="http://googletesting.blogspot.com/2012/09/conversation-with-test-engineer.html">what Alan Faulkner is doing</a> or ask the next Google Test Engineer you meet. Interested in joining the ranks of Test Engineers (or Software Engineers in Test)? Check out <a href="//goo.gl/2RDKj">http://goo.gl/2RDKj</a> About the Contributors: Albert Drona has been at Google for 5 years and is currently working on Google Maps for Mobile. Jatin Shah has been at Google for 9 months on Google+. Mohammad Khan has been at Google for 7 years and is currently working on Google+ releases. About the Author: Yvette Nameth has been at Google for 5 years and is currently working on Google Maps rendering. </div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </script> <noscript> <div dir="ltr" style="text-align: left;" trbidi="on"> By Yvette Nameth At Google, we’re very big into highlighting individuals’ strengths and using them to make teams and products better. However, we frequently get asked “What do Test Engineers (aka TEs) do?” I pause when I get this question since it’s hard to speak for my peers - I test <a href="http://google-latlong.blogspot.com/2011/07/evolving-look-of-google-maps-redux.html">Google Maps rendering</a>, which is just one small portion of what Google’s Test Engineers test. In order to get a clearer picture of what Test Engineers are responsible for, I chatted with three of my colleagues. We were able to identify the underlying Test Engineers’ similarities, while highlighting the differences. So what common themes do Test Engineers specialize in at Google? <div style="margin-left: 20px;"> We’re product experts: Test Engineers need to become a “go-to” person for how their product works and integrates with other Google products. (You aren’t expected to have this before working with a product, but you need to figure out how to become one on any product you work on!) TEs need to understand use cases and contracts with other services, products, and features. We aren’t expected to write unit tests for other engineer’s code; instead we ensure product quality on the functional and integration aspects of the product. We’re flexible: Test Engineers are required to switch tasks and re-prioritize frequently. From unplanned catastrophes, to shifting launch calendars, to people asking us questions, our work is filled with interrupts. We determine how to ensure quality in the face of the interrupts. We also modify our tests based on the pace of the development and understand that there is no one right way to test a product. Test Engineers adapt tools to meet their needs and understand when a tool just can’t get the job done. We’re clear communicators: We have to be able to communicate via test plans, design docs, bugs, email and code. Every day we work with a wide variety of people in different roles: Software Engineers, Software Engineers in Test, Product Managers, Usability Researchers, Designers, Legal Counsel, etc. We need to address these different audiences to make sure we’re either gathering the information that will help us build better strategies or presenting feedback that will help influence the product. We’re good at coordination: We are people who use our “in between the product and user” status to coordinate integration testing efforts between products. We may coordinate manual testing efforts by our manual testers; or we may make sure that test gaps are being addressed by “someone” (Test Engineer, Software Engineer in Test, or Software Engineer). We put our product knowledge and communication together with a bit of coordination and make sure that bugs are looked at and the product is getting tested hourly / daily. We have impact: Google Test Engineers have big impact. We hold responsibility thinking of ways that our products could fail in “real scenarios”; and then we add tests to make sure that the worst won’t come to pass. How big is this? Well, in my case, I’m responsible for making sure Google Maps represents a map that is useful to my relatives in the middle of rural Montana as well as my friends living in London, Paris or Sydney. When you add to that the billions of other users in different regions, speaking different languages and using the map for different reasons, I know that my testing is impacting their ability to get around and find out information about the physical world around them safely. We code: The other most common question is “Do you write code?” The answer is yes; Test Engineers at Google do code. </div> The three aspects that generally differentiate what a Test Engineer does day-to-day depend on the following: <div style="margin-left: 20px;"> Individual’s Strengths & Interests: Everyone is different and every TE has different passions, strengths and areas of expertise. Thankfully, Google’s a big enough company that many different areas of testing are available, and we gravitate to testing products we like. All TEs start with core competencies in testing, coding, and algorithms. How a TE applies this knowledge varies. The Type of Product: Desktop, web app or mobile? Frontend or backend? The technologies that our products use and run on create a lot of variation in what and how we test. The Product’s History / Lifecycle: Early concept products don’t resemble those that exist in production. And the amount of testing that a product already has will determine what testing the TE is focused on. We work creating a test roadmap that parallels the product’s development cycle and addresses any testing gaps. </div> If you still want to know what the day in the life of a Test Engineer entails, we’ll never be able to give you a general answer for that. Instead I suggest that you check out <a href="http://googletesting.blogspot.com/2012/09/conversation-with-test-engineer.html">what Alan Faulkner is doing</a> or ask the next Google Test Engineer you meet. Interested in joining the ranks of Test Engineers (or Software Engineers in Test)? Check out <a href="//goo.gl/2RDKj">http://goo.gl/2RDKj</a> About the Contributors: Albert Drona has been at Google for 5 years and is currently working on Google Maps for Mobile. Jatin Shah has been at Google for 9 months on Google+. Mohammad Khan has been at Google for 7 years and is currently working on Google+ releases. About the Author: Yvette Nameth has been at Google for 5 years and is currently working on Google Maps rendering. </div> <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/> </noscript> </div> </div> <div class='share'> <span class='twitter-custom social-wrapper' data-href='http://twitter.com/share?text=Google Testing Blog:Test Engineers @ Google&url=https://testing.googleblog.com/2013/01/test-engineers-google.html&via=googletesting'> <img alt='Share on Twitter' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png' width='24'/> <img alt='Share on Facebook' height='24' src='https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png' width='24'/> </div> <div class='comment-container'>  <a href='https://testing.googleblog.com/2013/01/test-engineers-google.html#comments' style='font-weight: 500; text-decoration: underline;'>20 comments</a> </div> <div class='post-footer'> <div class='cmt_iframe_holder' data-href='https://testing.googleblog.com/2013/01/test-engineers-google.html' data-viewtype='FILTERED_POSTMOD'></div> <a href='https://plus.google.com/112374322230920073195' rel='author' style='display:none;'> Google </a> <div class='label-footer'> Labels: <a class='label' href='https://testing.googleblog.com/search/label/Jobs' rel='tag'> Jobs </a> , <a class='label' href='https://testing.googleblog.com/search/label/Yvette%20Nameth' rel='tag'> Yvette Nameth </a> </div> </div> </div> <div class='blog-pager' id='blog-pager'> <a class='home-link' href='https://testing.googleblog.com/'>  </a> <a class='blog-pager-newer-link' href='https://testing.googleblog.com/search?updated-max=2014-04-01T02:00:00-07:00&max-results=5&reverse-paginate=true' id='Blog1_blog-pager-newer-link' title='Newer Posts'>  </a> <a class='blog-pager-older-link' href='https://testing.googleblog.com/search?updated-max=2013-01-14T11:27:00-08:00&max-results=5' id='Blog1_blog-pager-older-link' title='Older Posts'>  </a> </div> <div class='clear'></div> </div></div> </div> </div> <div class='col-right'> <div class='section' id='sidebar-top'><div class='widget HTML' data-version='1' id='HTML8'> <div class='widget-content'> <div class='searchBox'> <input type='text' title='Search This Blog' placeholder='Search blog ...' /> </div> </div> <div class='clear'></div> </div> </div> <div id='aside'> <div class='section' id='sidebar'><div class='widget Label' data-version='1' id='Label1'> <div class='tab'> <img class='sidebar-icon' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII='/> <h2> Labels </h2>  </div> <div class='widget-content list-label-widget-content'> <ul> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/TotT'> TotT </a> 104 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/GTAC'> GTAC </a> 61 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/James%20Whittaker'> James Whittaker </a> 42 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Misko%20Hevery'> Misko Hevery </a> 32 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Code%20Health'> Code Health </a> 31 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anthony%20Vallone'> Anthony Vallone </a> 27 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrick%20Copeland'> Patrick Copeland </a> 23 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jobs'> Jobs </a> 18 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Andrew%20Trenk'> Andrew Trenk </a> 13 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/C%2B%2B'> C++ </a> 11 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patrik%20H%C3%B6glund'> Patrik Höglund </a> 8 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/JavaScript'> JavaScript </a> 7 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Allen%20Hutchison'> Allen Hutchison </a> 6 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/George%20Pirocanac'> George Pirocanac </a> 6 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zhanyong%20Wan'> Zhanyong Wan </a> 6 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Harry%20Robinson'> Harry Robinson </a> 5 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Java'> Java </a> 5 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julian%20Harty'> Julian Harty </a> 5 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Bender'> Adam Bender </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alberto%20Savoia'> Alberto Savoia </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ben%20Yu'> Ben Yu </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Erik%20Kuefler'> Erik Kuefler </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Philip%20Zembrod'> Philip Zembrod </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shyam%20Seshadri'> Shyam Seshadri </a> 4 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome'> Chrome </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dillon%20Bly'> Dillon Bly </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Thomas'> John Thomas </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Lesley%20Katzen'> Lesley Katzen </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Kaplan'> Marc Kaplan </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Markus%20Clermont'> Markus Clermont </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Max%20Kanat-Alexander'> Max Kanat-Alexander </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sonal%20Shah'> Sonal Shah </a> 3 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/APIs'> APIs </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Abhishek%20Arya'> Abhishek Arya </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Myrvold'> Alan Myrvold </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alek%20Icev'> Alek Icev </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Android'> Android </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/April%20Fools'> April Fools </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chaitali%20Narla'> Chaitali Narla </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chris%20Lewis'> Chris Lewis </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Chrome%20OS'> Chrome OS </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Salas'> Diego Salas </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dori%20Reuveni'> Dori Reuveni </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Arbon'> Jason Arbon </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jochen%20Wuttke'> Jochen Wuttke </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kostya%20Serebryany'> Kostya Serebryany </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marc%20Eaddy'> Marc Eaddy </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marko%20Ivankovi%C4%87'> Marko Ivanković </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mobile'> Mobile </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Oliver%20Chang'> Oliver Chang </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simon%20Stewart'> Simon Stewart </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stefan%20Kennedy'> Stefan Kennedy </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Flakiness'> Test Flakiness </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Titus%20Winters'> Titus Winters </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tony%20Voellm'> Tony Voellm </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/WebRTC'> WebRTC </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yiming%20Sun'> Yiming Sun </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Yvette%20Nameth'> Yvette Nameth </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Zuri%20Kemp'> Zuri Kemp </a> 2 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Aaron%20Jacobs'> Aaron Jacobs </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Porter'> Adam Porter </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adam%20Raider'> Adam Raider </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Adel%20Saoud'> Adel Saoud </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alan%20Faulkner'> Alan Faulkner </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Alex%20Eagle'> Alex Eagle </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Amy%20Fu'> Amy Fu </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Anantha%20Keesara'> Anantha Keesara </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Antoine%20Picard'> Antoine Picard </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/App%20Engine'> App Engine </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ari%20Shamash'> Ari Shamash </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Arif%20Sukoco'> Arif Sukoco </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Benjamin%20Pick'> Benjamin Pick </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bob%20Nystrom'> Bob Nystrom </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Bruce%20Leban'> Bruce Leban </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Arguelles'> Carlos Arguelles </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Carlos%20Israel%20Ortiz%20Garc%C3%ADa'> Carlos Israel Ortiz García </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Cathal%20Weakliam'> Cathal Weakliam </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Christopher%20Semturs'> Christopher Semturs </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Clay%20Murphy'> Clay Murphy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dagang%20Wei'> Dagang Wei </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Maksimovich'> Dan Maksimovich </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Shi'> Dan Shi </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dan%20Willemsen'> Dan Willemsen </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Chen'> Dave Chen </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dave%20Gladfelter'> Dave Gladfelter </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Bendory'> David Bendory </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/David%20Mandelberg'> David Mandelberg </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Derek%20Snyder'> Derek Snyder </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Diego%20Cavalcanti'> Diego Cavalcanti </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Dmitry%20Vyukov'> Dmitry Vyukov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Eduardo%20Bravo%20Ortiz'> Eduardo Bravo Ortiz </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ekaterina%20Kamenskaya'> Ekaterina Kamenskaya </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliott%20Karpilovsky'> Elliott Karpilovsky </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Elliotte%20Rusty%20Harold'> Elliotte Rusty Harold </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Espresso'> Espresso </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Felipe%20Sodr%C3%A9'> Felipe Sodré </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Francois%20Aube'> Francois Aube </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Gene%20Volovich'> Gene Volovich </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Google%2B'> Google+ </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goran%20Petrovic'> Goran Petrovic </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Goranka%20Bjedov'> Goranka Bjedov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hank%20Duan'> Hank Duan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Havard%20Rast%20Blok'> Havard Rast Blok </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Hongfei%20Ding'> Hongfei Ding </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Elbaum'> Jason Elbaum </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jason%20Huggins'> Jason Huggins </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jay%20Han'> Jay Han </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Hoy'> Jeff Hoy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jeff%20Listfield'> Jeff Listfield </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jessica%20Tomechak'> Jessica Tomechak </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jim%20Reardon'> Jim Reardon </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joe%20Allan%20Muharsky'> Joe Allan Muharsky </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Joel%20Hynoski'> Joel Hynoski </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Micco'> John Micco </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/John%20Penix'> John Penix </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Rockway'> Jonathan Rockway </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Jonathan%20Velasquez'> Jonathan Velasquez </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Josh%20Armour'> Josh Armour </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Julie%20Ralph'> Julie Ralph </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kai%20Kent'> Kai Kent </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kanu%20Tewary'> Kanu Tewary </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Karin%20Lundberg'> Karin Lundberg </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kaue%20Silveira'> Kaue Silveira </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Bourrillion'> Kevin Bourrillion </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kevin%20Graney'> Kevin Graney </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kirkland'> Kirkland </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Kurt%20Alfred%20Kluever'> Kurt Alfred Kluever </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Manjusha%20Parvathaneni'> Manjusha Parvathaneni </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marek%20Kiszkis'> Marek Kiszkis </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Marius%20Latinis'> Marius Latinis </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Ivey'> Mark Ivey </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Manley'> Mark Manley </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mark%20Striebeck'> Mark Striebeck </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Matt%20Lowrie'> Matt Lowrie </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Meredith%20Whittaker'> Meredith Whittaker </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Bachman'> Michael Bachman </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Michael%20Klepikov'> Michael Klepikov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Aizatsky'> Mike Aizatsky </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mike%20Wacker'> Mike Wacker </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Mona%20El%20Mahdy'> Mona El Mahdy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Noel%20Yap'> Noel Yap </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Palak%20Bansal'> Palak Bansal </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Patricia%20Legaspi'> Patricia Legaspi </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Per%20Jacobsson'> Per Jacobsson </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Arrenbrecht'> Peter Arrenbrecht </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Peter%20Spragins'> Peter Spragins </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Norman'> Phil Norman </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Phil%20Rollet'> Phil Rollet </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Pooja%20Gupta'> Pooja Gupta </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Project%20Showcase'> Project Showcase </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Radoslav%20Vasilev'> Radoslav Vasilev </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Dewan'> Rajat Dewan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rajat%20Jain'> Rajat Jain </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Rich%20Martin'> Rich Martin </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Richard%20Bustamante'> Richard Bustamante </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Roshan%20Sembacuttiaratchy'> Roshan Sembacuttiaratchy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Ruslan%20Khamitov'> Ruslan Khamitov </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sam%20Lee'> Sam Lee </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sean%20Jordan'> Sean Jordan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sebastian%20D%C3%B6rner'> Sebastian Dörner </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Sharon%20Zhou'> Sharon Zhou </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Shiva%20Garg'> Shiva Garg </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Siddartha%20Janga'> Siddartha Janga </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Simran%20Basi'> Simran Basi </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stan%20Chan'> Stan Chan </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Stephen%20Ng'> Stephen Ng </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tejas%20Shah'> Tejas Shah </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Analytics'> Test Analytics </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Test%20Engineer'> Test Engineer </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tim%20Lyakhovetskiy'> Tim Lyakhovetskiy </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Tom%20O%27Neill'> Tom O'Neill </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/Vojta%20J%C3%ADna'> Vojta Jína </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/automation'> automation </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/dead%20code'> dead code </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/iOS'> iOS </a> 1 </li> <li> <a dir='ltr' href='https://testing.googleblog.com/search/label/mutation%20testing'> mutation testing </a> 1 </li> </ul> <div class='clear'></div> </div> </div><div class='widget BlogArchive' data-version='1' id='BlogArchive1'> <div class='tab'>  <h2> Archive </h2>  </div> <div class='widget-content'> <div id='ArchiveList'> <div id='BlogArchive1_ArchiveList'> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2025/'> 2025 </a> (1) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2025/01/'> Jan </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/'> 2024 </a> (13) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/04/'> Apr </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/03/'> Mar </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2024/02/'> Feb </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/'> 2023 </a> (14) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/12/'> Dec </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/10/'> Oct </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/09/'> Sep </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2023/04/'> Apr </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/'> 2022 </a> (2) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2022/02/'> Feb </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/'> 2021 </a> (3) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/06/'> Jun </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/04/'> Apr </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2021/03/'> Mar </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/'> 2020 </a> (8) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/12/'> Dec </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/08/'> Aug </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2020/05/'> May </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/'> 2019 </a> (4) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2019/01/'> Jan </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/'> 2018 </a> (7) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/07/'> Jul </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/05/'> May </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2018/02/'> Feb </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/'> 2017 </a> (17) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/07/'> Jul </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/02/'> Feb </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2017/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/'> 2016 </a> (15) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/10/'> Oct </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/09/'> Sep </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/04/'> Apr </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/03/'> Mar </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2016/02/'> Feb </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/'> 2015 </a> (14) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/10/'> Oct </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/08/'> Aug </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/06/'> Jun </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/03/'> Mar </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/02/'> Feb </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2015/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/'> 2014 </a> (24) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/12/'> Dec </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/11/'> Nov </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/10/'> Oct </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/09/'> Sep </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/08/'> Aug </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/07/'> Jul </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/06/'> Jun </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/03/'> Mar </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/02/'> Feb </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2014/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> ▼  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/'> 2013 </a> (16) <ul class='hierarchy'> <li class='archivedate expanded'> <a class='toggle' href='javascript:void(0)'> ▼  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/12/'> Dec </a> (1) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/12/the-google-test-and-development.html'> The Google Test and Development Environment - Pt. ... </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/11/'> Nov </a> (1) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/11/webrtc-audio-quality-testing.html'> WebRTC Audio Quality Testing </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/10/'> Oct </a> (1) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/10/espresso-for-android-is-here.html'> Espresso for Android is here! </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/08/'> Aug </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/08/how-google-team-tests-mobile-apps.html'> How the Google+ Team Tests Mobile Apps </a> </li> <li> <a href='https://testing.googleblog.com/2013/08/testing-on-toilet-test-behavior-not.html'> Testing on the Toilet: Test Behavior, Not Implemen... </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/07/'> Jul </a> (1) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/07/testing-on-toilet-know-your-test-doubles.html'> Testing on the Toilet: Know Your Test Doubles </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/06/'> Jun </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/06/testing-on-toilet-fake-your-way-to.html'> Testing on the Toilet: Fake Your Way to Better Tests </a> </li> <li> <a href='https://testing.googleblog.com/2013/06/optimal-logging.html'> Optimal Logging </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/05/'> May </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html'> Testing on the Toilet: Don’t Overuse Mocks </a> </li> <li> <a href='https://testing.googleblog.com/2013/05/gtac-2013-wrap-up.html'> GTAC 2013 Wrap-up </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/04/'> Apr </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/04/gtac-is-almost-here.html'> GTAC is Almost Here! </a> </li> <li> <a href='https://testing.googleblog.com/2013/04/two-new-videos-about-testing-at-google.html'> Two New Videos About Testing at Google </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/03/'> Mar </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/03/testing-on-toilet-testing-state-vs.html'> Testing on the Toilet: Testing State vs. Testing I... </a> </li> <li> <a href='https://testing.googleblog.com/2013/03/announcing-gtac-2013-agenda.html'> Announcing the GTAC 2013 Agenda </a> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2013/01/'> Jan </a> (2) <ul class='posts'> <li> <a href='https://testing.googleblog.com/2013/01/the-deadline-to-signup-for-gtac-is-jan.html'> The Deadline to Signup for GTAC is Jan 23 </a> </li> <li> <a href='https://testing.googleblog.com/2013/01/test-engineers-google.html'> Test Engineers @ Google </a> </li> </ul> </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/'> 2012 </a> (11) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/12/'> Dec </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/10/'> Oct </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/09/'> Sep </a> (1) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2012/08/'> Aug </a> (4) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/'> 2011 </a> (39) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/10/'> Oct </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/09/'> Sep </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/08/'> Aug </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/07/'> Jul </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/06/'> Jun </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/05/'> May </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/04/'> Apr </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/03/'> Mar </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/02/'> Feb </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2011/01/'> Jan </a> (3) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/'> 2010 </a> (37) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/12/'> Dec </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/11/'> Nov </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/10/'> Oct </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/09/'> Sep </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/08/'> Aug </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/07/'> Jul </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/04/'> Apr </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/03/'> Mar </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/02/'> Feb </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2010/01/'> Jan </a> (1) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/'> 2009 </a> (54) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/12/'> Dec </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/11/'> Nov </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/10/'> Oct </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/09/'> Sep </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/08/'> Aug </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/07/'> Jul </a> (15) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/06/'> Jun </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/05/'> May </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/04/'> Apr </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/02/'> Feb </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2009/01/'> Jan </a> (4) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/'> 2008 </a> (75) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/12/'> Dec </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/11/'> Nov </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/10/'> Oct </a> (9) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/09/'> Sep </a> (8) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/08/'> Aug </a> (9) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/07/'> Jul </a> (9) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/06/'> Jun </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/05/'> May </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/04/'> Apr </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/03/'> Mar </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/02/'> Feb </a> (4) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2008/01/'> Jan </a> (2) </li> </ul> </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/'> 2007 </a> (41) <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/10/'> Oct </a> (6) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/09/'> Sep </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/08/'> Aug </a> (3) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/07/'> Jul </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/06/'> Jun </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/05/'> May </a> (2) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/04/'> Apr </a> (7) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/03/'> Mar </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/02/'> Feb </a> (5) </li> </ul> <ul class='hierarchy'> <li class='archivedate collapsed'> <a class='toggle' href='javascript:void(0)'> ►  </a> <a class='post-count-link' href='https://testing.googleblog.com/2007/01/'> Jan </a> (4) </li> </ul> </li> </ul> </div> </div> <div class='clear'></div> </div> </div><div class='widget HTML' data-version='1' id='HTML6'> <div class='widget-content'> <a href="http://googletesting.blogspot.com/atom.xml"> <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==" class="sidebar-icon" /> <h2>Feed</h2> </a> </div> <div class='clear'></div> </div></div> <div class='section' id='sidebar-bottom'><div class='widget HTML' data-version='1' id='HTML9'> <div class='widget-content'> <a href='http://cloud.feedly.com/#subscription%2Ffeed%2Fhttp%3A%2F%2Fgoogletesting.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='blank'><img id="feedlyFollow" src="https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_s6wiCbRaIkeKyIm5yDvPyujQdPWn0hWbr8qVwudX_oR2c20jR-G3wygGM_tcuRIjKkl28rkIh9y331lObTjNRHFhf7e_f2XNuziIcuAe9IT1eYe-cfe88iejoXhPl7FpOdkQ8BMjfeCRkZKiFHmY0=s0-d" alt="follow us in feedly" width="66" height="20"></a> <div class="share followgooglewrapper"> <button data-href="https://twitter.com/intent/follow?original_referer=http://googletesting.blogspot.com/&screen_name=googletesting" onclick='sharingPopup(this);' id='twitter-share'>Follow @googletesting</button> <script> function sharingPopup (button) { var url = button.getAttribute("data-href"); window.open( url,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); } </script> </div> </div> <div class='clear'></div> </div></div> </div> </div> <div style='clear:both;'></div> </div>  <div class='google-footer-outer loading'> <div id='google-footer'> <a href='//www.google.com/'> <img class='google-logo-dark' height='36' src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==' style='margin-top: -16px;' width='92'/> </a> <ul> <li> <a href='//www.google.com/'> Google </a> </li> <li> <a href='//www.google.com/policies/privacy/'> Privacy </a> </li> <li> <a href='//www.google.com/policies/terms/'> Terms </a> </li> </ul> </div> </div> <script type='text/javascript'> //<![CDATA[ // Social sharing popups. var postEl = document.getElementsByClassName('social-wrapper'); var postCount = postEl.length; for(i=0; i<postCount;i++){ postEl[i].addEventListener("click", function(event){ var postUrl = this.getAttribute("data-href"); window.open( postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes'); });} //]]> </script> <script type='text/javascript'> //<![CDATA[ var BreakpointHandler = function() { this.initted = false; this.isHomePage = false; this.isMobile = false; }; BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) { // Use $.trim for IE8 compatibility summaryHtml = $.trim(summaryHtml).replace(/( |\s)+$/,''); if (lastNode.nodeType == 3) { var lastChar = summaryHtml.slice(-1); if (!lastChar.match(/[.”"?]/)) { if (!lastChar.match(/[A-Za-z]/)) { summaryHtml = summaryHtml.slice(0, -1); } summaryHtml += ' ...'; } } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) { summaryHtml += ' ...'; } return summaryHtml; }; BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) { var seenWords = 0; var summaryHtml = ''; for (var i=0; i < content.childNodes.length; i++) { var node = content.childNodes[i]; var nodeText; if (node.nodeType == 1) { if (node.hasAttribute('data-about-pullquote')) { continue; } nodeText = node.textContent; if (nodeText === undefined) { // innerText for IE8 nodeText = node.innerText; } if (node.nodeName == 'DIV' || node.nodeName == 'B') { // Don't end early if we haven't seen enough words. if (seenWords < 10) { continue; } if (i > 0) { summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]); } break; } summaryHtml += node.outerHTML; } else if (node.nodeType == 3) { nodeText = node.nodeValue; summaryHtml += nodeText + ' '; } var words = nodeText.match(/\S+\s*/g); if (!words) { continue; } var remain = numWords - seenWords; if (words.length >= remain) { summaryHtml = this.finalizeSummary(summaryHtml, node); break; } seenWords += words.length; } return summaryHtml; }; BreakpointHandler.prototype.detect = function() { var match, pl = /\+/g, search = /([^&=]+)=?([^&]*)/g, decode = function (s) { return decodeURIComponent(s.replace(pl, " ")); }, query = window.location.search.substring(1); var urlParams = {}; while (match = search.exec(query)) urlParams[decode(match[1])] = decode(match[2]); this.isListPage = $('html').hasClass('list-page'); this.isMobile = urlParams['m'] === '1'; this.isHomePage = window.location.pathname == '/'; }; BreakpointHandler.prototype.initContent = function() { var self = this; $('.post').each(function(index) { var body = $(this).children('.post-body')[0]; var content = $(body).children('.post-content')[0]; $(content).addClass('post-original'); var data = $(content).children('script').html(); data = self.rewriteForSSL(data); // If exists, extract specified editor's preview. var match = data.match(/([\s\S]+?)<div data-is-preview.+?>([\s\S]+)<\/div>/m); if (match) { data = match[1]; } // Prevent big images from loading when they aren't needed. // This must be done as a pre-injection step, since image loading can't be // canceled once embedded into the DOM. if (self.isListPage && self.isMobile) { data = data.replace(/<(img|iframe) .+?>/g, ''); } // Insert template to be rendered as nodes. content.innerHTML = data; if (self.isListPage) { var summary = document.createElement('div'); $(summary).addClass('post-content'); $(summary).addClass('post-summary'); body.insertBefore(summary, content); if (match) { // Use provided summary. summary.innerHTML = match[2]; } else { // Generate a summary. // Summary generation relies on DOM, so it must occur after content is // inserted into the page. summary.innerHTML = self.generateSummaryFromContent(content, 30); } // Add read more link to summary. var titleAnchor = $(this).find('.title a')[0]; var link = titleAnchor.cloneNode(true); link.innerHTML = 'Read More'; $(link).addClass('read-more'); summary.appendChild(link); } }); // Firefox does not allow for proper styling of BR. if (navigator.userAgent.indexOf('Firefox') > -1) { $('.post-content br').replaceWith(''); } $('.loading').removeClass('loading'); }; BreakpointHandler.prototype.process = function() { if (!this.initted) { var makeInsecureImageRegex = function(hosts) { var whitelist = hosts.join('|').replace(/\./g,'\\.'); // Normal image tags, plus input images (yes, this is possible!) return new RegExp('(<(img|input)[^>]+?src=("|\'))http:\/\/(' + whitelist +')', 'g'); }; this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS); this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]); this.detect(); this.initContent(); this.initted = true; } }; BreakpointHandler.KNOWN_HTTPS_HOSTS = [ "www.google.org", "www.google.com", "services.google.com", "blogger.com", "draft.blogger.com", "www.blogger.com", "photos1.blogger.com", "photos2.blogger.com", "photos3.blogger.com", "blogblog.com", "img1.blogblog.com", "img2.blogblog.com", "www.blogblog.com", "www1.blogblog.com", "www2.blogblog.com", "0.bp.blogspot.com", "1.bp.blogspot.com", "2.bp.blogspot.com", "3.bp.blogspot.com", "4.bp.blogspot.com", "lh3.googleusercontent.com", "lh4.googleusercontent.com", "lh5.googleusercontent.com", "lh6.googleusercontent.com", "themes.googleusercontent.com", ]; BreakpointHandler.prototype.rewriteForSSL = function(html) { // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content. return html.replace(this.sslImageRegex, '$1https://$4') .replace(this.sslImageCurrentDomainRegex, '$1//$4') .replace(/(<(embed|iframe)[^>]+?src=("|'))http:\/\/([^"']*?(youtube|picasaweb\.google)\.com)/g, '$1https://$4') // Slideshow SWF takes a image host, so we need to rewrite that parameter. .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s'); }; $(document).ready(function() { var handler = new BreakpointHandler(); handler.process(); // Top-level navigation. $(".BlogArchive .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); $(".Label .tab").click(function(ev) { ev.preventDefault(); $(this).parent().toggleClass('active'); $(this).siblings().slideToggle(300); }); // Blog archive year expansion. $('.BlogArchive .intervalToggle').click(function(ev) { ev.preventDefault(); if ($(this).parent().hasClass('collapsed')) { $(this).parent().removeClass('collapsed'); $(this).parent().addClass('expanded'); } else { $(this).parent().removeClass('expanded'); $(this).parent().addClass('collapsed'); } }); // Reverse order of months. $('.BlogArchive .intervalToggle + div').each(function(_, items) { var year = $(this); year.children().each(function(_, month) { year.prepend(month); }); }); // Set anchors to open in new tab. $('.post-content img').parent().each(function(_, node) { if (node.nodeName == 'A') { $(this).attr('target', '_blank'); } }); // Process search requests. $('.searchBox input').on("keypress", function(ev) { if (ev.which == 13) { window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val()); } }); }); (function($, window) { var archiveButton = $($('#sidebar .widget.BlogArchive h2')[0]); var folderIcon = $('#sidebar .widget.BlogArchive h2::after'); var archivePanel = $('#BlogArchive1'); archiveButton.click(function(e) { if (archivePanel.hasClass('archive-open')) { // It's open, so now we close it archivePanel.removeClass('archive-open'); archivePanel.addClass('archive-closed'); archiveButton.removeClass('archive-open'); archiveButton.addClass('archive-closed'); archivePanel.css('height', '60px'); } else { // It's closed, so open it archivePanel.removeClass('archive-closed'); archivePanel.addClass('archive-open'); archiveButton.removeClass('archive-closed'); archiveButton.addClass('archive-open'); archivePanel.css('height', 'auto'); folderIcon.css('content', 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEggkDzi8QVSua_XxLOnfxWE_8nQc1MAhRPBxJej91JW2ZBbKvb2TjaNf66ZoTqCrBivwRr-TLJYxFm2ZAiMd-zfbJ0TjblXGvUszWFbdRncNOL7jseiJCL9uQoqN8BM8FpesdSw/s1600/keyboard_arrow_up_grey600_24dp.png'); } }); })($, window); (function ($, window, devsite) { devsite.devsite = {}; devsite.expandLeftNav = devsite.expandLeftNav || {}; devsite.expandLeftNav.ANALYTICS_LABEL_ = 'Hamburger menu'; devsite.expandLeftNav.ANALYTICS_ACTION_OPEN_ = 'Open'; devsite.expandLeftNav.ANALYTICS_ACTION_CLOSE_ = 'Close'; devsite.expandLeftNav.lastState_ = 0; devsite.expandLeftNav.chekovEnabled_ = false; devsite.expandLeftNav.init = function () { devsite.expandLeftNav.chekovEnabled_ = true; var navElement; if (devsite.expandLeftNav.chekovEnabled_) { navElement = $('.devsite-nav-responsive'); } else { navElement = $('.devsite-section-nav-responsive'); } this.drawerWidth = parseInt(navElement.css('width')); navElement.css({ 'left': -(this.drawerWidth) }); var expandButton = $('.devsite-expand-section-nav'); expandButton.click(function () { devsite.expandLeftNav.handleNavigationOpened(); }); navElement.find('.devsite-nav-responsive-forward').click( devsite.expandLeftNav.openPanel); navElement.find('.devsite-nav-responsive-back').click( devsite.expandLeftNav.closePanel); if ($('.devsite-nav-responsive-tabs-panel + ' + '.devsite-nav-responsive-sidebar-panel').length) { devsite.expandLeftNav.openPanel(); } }; devsite.expandLeftNav.handleNavigationOpened = function (opt_noAnimate) { var mask; if (devsite.expandLeftNav.chekovEnabled_) { var nav = $('.devsite-nav-responsive'); if (opt_noAnimate) { nav.addClass('devsite-nav-responsive-no-animate'); mask = devsite.devsite.showSiteMask(0); } else { mask = devsite.devsite.showSiteMask(); } nav.addClass('devsite-nav-responsive-open'); } else { devsite.expandLeftNav.lastState_ = devsite.sticky.currentState; devsite.sticky.setState(devsite.sticky.state.COLLAPSED_HEADER); var headerHeight = devsite.sticky.getHeaderHeight() - devsite.sticky.desiredMargin; var drawerHeight = $(window).height() - headerHeight; mask = devsite.devsite.showArticleMask(); $('.devsite-section-nav-responsive').css({ 'height': drawerHeight, 'left': '0', 'top': headerHeight, 'visibility': 'visible' }).focus(); } $(mask).click(devsite.expandLeftNav.handleNavigationClosed); $(document).on('keydown.escape', function (e) { if (e.keyCode == $.ui.keyCode.ESCAPE) { devsite.expandLeftNav.handleNavigationClosed(); $(document).off('keydown.escape'); } }); }; devsite.expandLeftNav.getScrollTop = function () { return $(window).scrollTop(); }; devsite.expandLeftNav.handleNavigationClosed = function () { var nav; if (devsite.expandLeftNav.chekovEnabled_) { nav = $('.devsite-nav-responsive'); devsite.devsite.hideSiteMask(); nav.removeClass('devsite-nav-responsive-open ' + 'devsite-nav-responsive-no-animate'); } else { nav = $('.devsite-section-nav-responsive'); devsite.devsite.hideArticleMask(); devsite.sticky.setState(devsite.expandLeftNav.lastState_); nav .css('left', -(devsite.expandLeftNav.drawerWidth)) .one('transitionend', function () { nav.css('visibility', 'hidden'); }); } nav.scrollTop(0); }; devsite.expandLeftNav.openPanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel.show(); parentPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel .removeClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); parentPanel.hide(); }); setTimeout(function () { childPanel .addClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); }, 1); }; devsite.expandLeftNav.closePanel = function () { var parentPanel = $('.devsite-nav-responsive-tabs-panel'); var childPanel = $('.devsite-nav-responsive-sidebar-panel'); childPanel .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); parentPanel .show() .removeClass('devsite-nav-responsive-transition') .addClass('devsite-nav-responsive-transform'); setTimeout(function () { parentPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform'); childPanel .addClass('devsite-nav-responsive-transition') .removeClass('devsite-nav-responsive-transform') .one('transitionend', function () { childPanel.hide(); parentPanel.removeClass('devsite-nav-responsive-transition'); }); }, 1); }; devsite.expandLeftNav.FADE_SLOW_ = 'slow'; devsite.expandLeftNav.FADE_FAST_ = 'fast'; devsite.expandLeftNav.SITE_MASK_CSS_ = '.devsite-site-mask'; devsite.devsite.showSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_SLOW_; } devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, false, opt_animate); }; devsite.devsite.hideSiteMask = function(opt_animate) { if (opt_animate === undefined) { opt_animate = devsite.expandLeftNav.FADE_FAST_; } devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_(devsite.expandLeftNav.SITE_MASK_CSS_, true, opt_animate); }; devsite.devsite.showArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(false); return devsite.devsite.setMask_('.devsite-article-mask', false, devsite.expandLeftNav.FADE_SLOW_); }; devsite.devsite.hideArticleMask = function() { devsite.devsite.setMouseScrollingEnabled(true); return devsite.devsite.setMask_('.devsite-article-mask', true, devsite.expandLeftNav.FADE_FAST_); }; devsite.devsite.setMask_ = function(className, out, opt_fadeTime) { var query = $(className); if (opt_fadeTime === 0) { out ? query.hide() : query.show(); } else { out ? query.fadeOut(opt_fadeTime) : query.fadeIn(opt_fadeTime); } return $(className)[0]; }; devsite.devsite.setMouseScrollingEnabled = function(trueOrFalse) { if (trueOrFalse == true) { $('html, body').css({ 'overflow': '' }); } else { $('html, body').css({ 'overflow': 'hidden' }); } }; })($, window, devsite = {}); if (window.jQuery) { $(document).ready(function () { if (window.devsite) { devsite.expandLeftNav.init(); } }); } //]]> </script> <style> .widget ul{ line-height: 1.6 !important; } #sidebar ul li a{ color:black; line-height: 20px; } #sidebar ul li a:hover{ color:#4184F3; } </style> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/60983134-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY5lo-GvzTKjeVBe7Vynasa2BoVXzg:1739707693695';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d15045980','//testing.googleblog.com/2013/','15045980'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '15045980', 'title': 'Google Testing Blog', 'url': 'https://testing.googleblog.com/2013/', 'canonicalUrl': 'https://testing.googleblog.com/2013/', 'homepageUrl': 'https://testing.googleblog.com/', 'searchUrl': 'https://testing.googleblog.com/search', 'canonicalHomepageUrl': 'https://testing.googleblog.com/', 'blogspotFaviconUrl': 'https://testing.googleblog.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'G-838ZCPQWM6', 'analytics4': true, 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Google Testing Blog - RSS\x22 href\x3d\x22https://testing.googleblog.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Google Testing Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/15045980/posts/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/32c3d108bdd93523', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'archive', 'pageName': '2013', 'pageTitle': 'Google Testing Blog: 2013'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Google Testing Blog', 'description': '', 'url': 'https://testing.googleblog.com/2013/', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': true, 'isLabelSearch': false, 'archive': {'year': 2013, 'rangeMessage': 'Showing posts from 2013'}}}]); _WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/918196653-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML9', 'sidebar-bottom', document.getElementById('HTML9'), {}, 'displayModeFull')); </script> </body> </html>