CINXE.COM

<!DOCTYPE html> <html xmlns="https://www.w3.org/1999/xhtml" lang="en"> <head>  <script async src="https://www.googletagmanager.com/gtag/js?id=G-RHQHCVMDX3"></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-RHQHCVMDX3'); // new G4 ID gtag('config', 'G-7XLBH2NC54'); // combo gtag('config', 'UA-111316006-20'); // old UA ID </script> <meta charset="UTF-8"/> <meta content="IE=Edge" http-equiv="X-UA-Compatible"/> <meta content="width=device-width, initial-scale=1.0" name="viewport"/> <meta content="a5327bf28169772478eb3d2240e2b145" name="pageID"/><link href="https://weinberg.northwestern.edu/common/v2/css/styles.css" media="screen" rel="stylesheet" type="text/css"/><link href="https://weinberg.northwestern.edu/common/v2/css/print.css" media="print" rel="stylesheet" type="text/css"/><link href="https://weinberg.northwestern.edu/common/v2/css/swiper.css" media="screen" rel="stylesheet" type="text/css"/> <link href="../../css/department.css" media="screen" rel="stylesheet" type="text/css"/> <title>Mechanical Turk, Tim Brady: Cognitive Science Program - Northwestern University</title> <link href="https://cogsci.northwestern.edu/about/resources/mechanical-turk-tim-brady.html" rel="canonical"/> <meta content="https://cogsci.northwestern.edu/about/resources/mechanical-turk-tim-brady.html" property="og:url"/> <meta content="Mechanical Turk, Tim Brady: Cognitive Science Program - Northwestern University" property="og:title"/> <meta content="//common.northwestern.edu/v8/images/northwestern-thumbnail.jpg" property="og:image"/> </head> <body> <a id="top"></a> <script>document.getElementsByTagName('body')[0].setAttribute('class','standard-page');</script> <a class="screen-reader-shortcut" href="#main-content">Skip to main content</a>  <header> <div id="top-bar"> <div class="contain-1120"> <div id="left"><a href="https://www.northwestern.edu/"><img alt="Northwestern University" src="https://weinberg.northwestern.edu/common/v2/css/images/northwestern-university.svg"/></a> <div id="school"><a href="https://weinberg.northwestern.edu/">Weinberg College <span>of <strong>Arts & Sciences</strong></span></a></div> </div> </div> </div> <div id="global-links"></div> <div class="bottom-bar contain-1120"><div id="department"><h1><a href="../../index.html">COGNITIVE SCIENCE PROGRAM</a></h1></div> <div class="hide-mobile" id="search"> <div class="search-form"> <form action="https://search.northwestern.edu/" method="get" role="search"> <label class="hide-label" for="q-desktop">Search this site</label> <input id="q-desktop" name="q" placeholder="Search this site" type="text"/> <input name="as_sitesearch" type="hidden" value="https://cogsci.northwestern.edu"/> <input name="sitetitle" type="hidden" value="Cognitive Science Program"/> <button type="submit"><span class="hide-label">Search</span></button> </form> </div> </div> </div> <div id="mobile-links"><a class="mobile-link mobile-nav-link" href="#mobile-nav"><span class="hide-label">Menu</span></a> <nav id="mobile-nav"><ul aria-label="main menu"><li><a class="active" href="../index.html">About</a><span class="arrow open"><a aria-haspopup="true" href="#" role="button"><span>Collapse</span>About Submenu</a></span><ul aria-expanded="true" aria-hidden="false" style="display:block"><li><a href="../affiliated-departments.html">Affiliated Departments</a></li><li><a class="active" href="index.html">Resources</a><span class="arrow open"><a aria-haspopup="true" href="#" role="button"><span>Collapse</span>Resources Submenu</a></span><ul aria-expanded="true" aria-hidden="false" style="display:block"><li><a href="localResources.html">Local Talks & Groups</a></li><li><a class="active" href="mechanical-turk-tim-brady.html">Mechanical Turk</a></li></ul></li><li><a href="../news/index.html">News</a></li></ul></li><li><a href="../../events/index.html">Events</a><span class="arrow"><a aria-haspopup="true" href="#" role="button"><span>Expand</span>Events Submenu</a></span><ul aria-expanded="false" aria-hidden="true"><li><a href="../../events/workshops-and-tutorials/index.html">Previous Workshops and Tutorials</a></li></ul></li><li><a href="../../undergraduate/index.html">Undergraduate</a><span class="arrow"><a aria-haspopup="true" href="#" role="button"><span>Expand</span>Undergraduate Submenu</a></span><ul aria-expanded="false" aria-hidden="true"><li><a href="../../undergraduate/first-year-focus.html">First-Year Focus</a></li><li><a href="../../undergraduate/course-information.html">Course Information</a></li><li><a href="../../undergraduate/undergraduate-advising.html">Advising</a></li><li><a href="../../undergraduate/research.html">Research Opportunities</a></li><li><a href="../../undergraduate/honors.html">Honors in Cognitive Science</a></li><li><a href="../../undergraduate/undergraduate-award.html">Undergraduate Award</a></li><li><a href="../../undergraduate/petition-to-graduate.html">Petition to Graduate</a></li><li><a href="../../undergraduate/surf.html">Summer Research Fellowships</a></li><li><a href="../../undergraduate/faq.html">FAQs</a></li><li><a href="../../undergraduate/independent-study.html">Independent Study</a></li><li><a href="../../undergraduate/cog_sci_club.html">Cog Sci Club</a></li></ul></li><li><a href="../../graduate/index.html">Graduate</a><span class="arrow"><a aria-haspopup="true" href="#" role="button"><span>Expand</span>Graduate Submenu</a></span><ul aria-expanded="false" aria-hidden="true"><li><a href="../../graduate/graduate-certificate/index.html">Graduate Certificate</a><span class="arrow"><a aria-haspopup="true" href="#" role="button"><span>Expand</span>Graduate Certificate Submenu</a></span><ul aria-expanded="false" aria-hidden="true"><li><a href="../../graduate/graduate-certificate/graduating-as-a-specialist.html">Certificate Completion </a></li><li><a href="../../graduate/graduate-certificate/eligible-courses.html">Eligible Courses</a></li></ul></li><li><a href="../../graduate/funding_for_graduate_specialists/index.html">Funding for Graduate Specialists</a><span class="arrow"><a aria-haspopup="true" href="#" role="button"><span>Expand</span>Funding for Graduate Specialists Submenu</a></span><ul aria-expanded="false" aria-hidden="true"><li><a href="../../graduate/funding_for_graduate_specialists/advanced_fellowship_details.html">Advanced Fellowship Details</a></li><li><a href="../../graduate/funding_for_graduate_specialists/travel_grants.html">Travel Grants</a></li></ul></li></ul></li><li><a href="../../people/index.html">People</a><span class="arrow"><a aria-haspopup="true" href="#" role="button"><span>Expand</span>People Submenu</a></span><ul aria-expanded="false" aria-hidden="true"><li><a href="../../people/index.html">Affiliated Faculty</a></li><li><a href="../../people/program-committee.html">Program Committee</a></li><li><a href="../../people/staff.html">Staff</a></li><li><a href="../../people/student-advisory-board.html">Student Advisory Board</a></li></ul></li></ul> <div id="mobile-nav-bottom"></div> </nav> <a class="mobile-link mobile-search-link" href="#mobile-search"><span class="hide-label">Search</span></a> <div id="mobile-search"> <div class="search-form"> <form action="https://search.northwestern.edu/" method="get" role="search"> <label class="hide-label" for="q-mobile">Search this site</label> <input id="q-mobile" name="q" placeholder="Search this site" type="text"/> <input name="as_sitesearch" type="hidden" value="https://cogsci.northwestern.edu"/> <input name="sitetitle" type="hidden" value="Cognitive Science Program"/> <button type="submit"><span class="hide-label">Search</span></button> </form> </div> </div> </div> </header> <nav aria-label="main navigation menu" class="narrow-dropdown" id="top-nav"><div class="contain-1120"><ul role="menubar"><li class="active"><a href="../index.html">About <span class="dropdown-arrow"></span></a><ul class="dropdown"><li class="nav-links"><ul><li><a href="../affiliated-departments.html">Affiliated Departments</a></li><li><a href="index.html">Resources</a></li><li><a href="../news/index.html">News</a></li></ul></li></ul></li> <li><a href="../../events/index.html">Events <span class="dropdown-arrow"></span></a><ul class="dropdown"><li class="nav-links"><ul><li><a href="../../events/workshops-and-tutorials/index.html">Previous Workshops and Tutorials</a></li></ul></li></ul></li> <li><a href="../../undergraduate/index.html">Undergraduate <span class="dropdown-arrow"></span></a><ul class="dropdown"><li class="nav-links"><ul><li><a href="../../undergraduate/first-year-focus.html">First-Year Focus</a></li><li><a href="../../undergraduate/course-information.html">Course Information</a></li><li><a href="../../undergraduate/undergraduate-advising.html">Advising</a></li><li><a href="../../undergraduate/research.html">Research Opportunities</a></li><li><a href="../../undergraduate/honors.html">Honors in Cognitive Science</a></li><li><a href="../../undergraduate/undergraduate-award.html">Undergraduate Award</a></li><li><a href="../../undergraduate/petition-to-graduate.html">Petition to Graduate</a></li><li><a href="../../undergraduate/surf.html">Summer Research Fellowships</a></li><li><a href="../../undergraduate/faq.html">FAQs</a></li><li><a href="../../undergraduate/independent-study.html">Independent Study</a></li><li><a href="../../undergraduate/cog_sci_club.html">Cog Sci Club</a></li></ul></li></ul></li> <li><a href="../../graduate/index.html">Graduate <span class="dropdown-arrow"></span></a><ul class="dropdown nav-align-right"><li class="nav-links"><ul><li><a href="../../graduate/graduate-certificate/index.html">Graduate Certificate</a></li><li><a href="../../graduate/funding_for_graduate_specialists/index.html">Funding for Graduate Specialists</a></li></ul></li></ul></li> <li><a href="../../people/index.html">People <span class="dropdown-arrow"></span></a><ul class="dropdown nav-align-right"><li class="nav-links"><ul><li><a href="../../people/index.html">Affiliated Faculty</a></li><li><a href="../../people/program-committee.html">Program Committee</a></li><li><a href="../../people/staff.html">Staff</a></li><li><a href="../../people/student-advisory-board.html">Student Advisory Board</a></li></ul></li></ul></li> </ul></div></nav> <div id="page"><ul id="breadcrumbs"><li><a href="../../index.html">Home</a></li><li><a href="../index.html">About</a></li><li><a href="index.html">Resources</a></li><li class="active">Mechanical Turk</li></ul> <nav aria-label="section navigation menu" id="left-nav" tabindex="-1"><h2><a href="../index.html">About</a></h2><ul><li><a href="../affiliated-departments.html">Affiliated Departments</a></li><li><a class="active" href="index.html">Resources</a><ul class="open"><li><a href="localResources.html">Local Talks & Groups</a></li><li><a class="active" href="mechanical-turk-tim-brady.html">Mechanical Turk</a></li></ul></li><li><a href="../news/index.html">News</a></li></ul> </nav> <main id="main-content" class="content" tabindex="-1"> <h2>Mechanical Turk, Tim Brady</h2><div id="sidebar"><div class="box">More information, including <span><span class="Apple-converted-space"></span>a bunch of code for more advanced techniques is available at Dr. Brady's website:</span> <a data-saferedirecturl="https://www.google.com/url?hl=en&q=http://timbrady.org/ttt&source=gmail&ust=1478499477900000&usg=AFQjCNG9ry23nEkpYd6jCIX84paO0DZ8zQ" href="http://timbrady.org/ttt" rel="noopener" target="_blank">http://timbrady.org/ttt</a><span><span class="Apple-converted-space"><br/> </span></span></div></div><p><span class="SPEAKER"><strong>Dr. Tim Brady<span class="Apple-converted-space"> </span></strong><br/>Psychology Department<span class="Apple-converted-space"> </span><br/>University of California, San Diego<span class="Apple-converted-space"> </span><br/> </span> <span></span><span class="SPEAKER"><a href="http://timbrady.org/">More about Dr. Brady's research<span class="Apple-converted-space"> </span></a></span><span></span><span class="SPEAKER"></span></p> <ul> <li><span class="SPEAKER"><span class="Apple-converted-space">Video of Dr. Brady's introductory talk (running time one hour) <a href="https://northwestern.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=c355dcba-348f-4069-98c5-d572e8de77b4" rel="noopener" target="_blank">is available here.</a></span></span></li> <li><span class="SPEAKER"><span class="Apple-converted-space">Video of his in-depth Tutorial (running time 3.5 hours) <a href="https://northwestern.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=246a910a-3277-4f85-a2da-d9c95c215aa1" rel="noopener" target="_blank">is available here.</a></span></span></li> </ul> <h3><span class="SPEAKER"><span class="SPEAKER"></span></span>Running experiments on Mechanical Turk</h3> <blockquote>Mechanical Turk is a service provided Amazon that enables anybody to post short tasks and have participants from around the United States or the world complete them in exchange for payment. Workers can browse a huge set of tasks and choose which to complete. Only a small proportion of tasks on Turk are cognitive science experiments: the vast majority are short, simple tasks for business purposes (e.g., translating or transcribing speech; screening for inappropriate images, etc).<span class="Apple-converted-space"> </span><br/> <br/> In this talk, I'll give a brief overview of using Mechanical Turk as a large-scale subject pool for research. I'll review the pros and cons of Turk as a participant sample and as a platform for experiments. I'll review what we know about the demographics of Turk users and what we know about the validity of Turk for psychology experiments. I'll then explore the technical and practical limitations of running online experiments, and give a variety of examples of Turk studies, both from my own work (e.g., Brady & Alvarez, 2011, Psych. Science; Brady & Tenenbaum, 2013, Psych. Review) and from others.</blockquote> </main> </div> <footer> <div class="contain-970"> <div class="footer-content"><a href="https://www.northwestern.edu"><img alt="Northwestern University" src="https://common.northwestern.edu/v8/css/images/northwestern-university.svg"/></a> <ul> <li>© <script>// <![CDATA[ document.write(new Date().getFullYear()) // ]]></script> Northwestern University</li> <li><a href="https://www.northwestern.edu/fm/campus/building-access/index.html">Building Access</a></li> <li><a href="https://www.northwestern.edu/emergency/index.html">Campus Emergency Information</a></li> <li><a href="https://hr.northwestern.edu/careers/">Careers</a></li> <li><a href="https://www.northwestern.edu/contact.html">Contact Northwestern University</a></li> <li><a href="https://www.northwestern.edu/disclaimer.html">Disclaimer</a></li> <li><a href="https://www.northwestern.edu/accessibility/about/report-an-accessibility-issue.html">Report an Accessibility Issue</a></li> <li><a href="http://policies.northwestern.edu/">University Policies</a></li> </ul></div> <div class="footer-content contact"><ul><li class="footer-pin-icon"><span class="hide-label">Address</span></li><li><strong>Cognitive Science Program</strong></li><li>Room 214, Swift Hall, 2029 Sheridan Road</li><li>Evanston, IL 60208</li></ul><ul><li class="footer-phone-icon"><span class="hide-label">Phone number</span></li><li><strong>Phone</strong></li><li> 847.467.2035</li></ul><ul><li><strong>Fax</strong></li><li>847.491.7859</li></ul><ul><li class="footer-email-icon"><span class="hide-label">Email Address</span></li><li><a href="mailto:cogsci@northwestern.edu">cogsci@northwestern.edu</a></li></ul></div> <div class="footer-content"><p><strong>Social Media</strong></p><a class="social twitter" href="https://twitter.com/nucogsci">Twitter</a><a class="social we-will" href="http://wewill.northwestern.edu/weinbergcollege">We Will</a></div> <div class="footer-content"><ul> <li><a href="https://www.northwestern.edu/transportation-parking/">Parking</a></li> <li><a href="https://maps.northwestern.edu/">Maps</a></li> <li><a href="https://www.northwestern.edu/transportation-parking/shuttles/routes/">Shuttles</a></li> <li><a href="https://offices.northwestern.edu/">Directory</a></li> <li><a href="https://www.northwestern.edu/norris/services/bookstore/">Bookstore</a></li> <li><a href="http://www.library.northwestern.edu/">Library</a></li> </ul></div> </div> </footer> <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js" type="text/javascript"></script> <script src="https://weinberg.northwestern.edu/common/v2/js/scripts.js" type="text/javascript"></script> </body> </html>