CINXE.COM

<!doctype html> <html lang="en" dir="ltr" class="plugin-pages plugin-id-default" data-has-hydrated="false"> <head> <meta charset="UTF-8"> <meta name="generator" content="Docusaurus v3.0.0-rc.1"> <title data-rh="true">NodeJs SuperAgent - Setting Fake User-Agents | ScrapeOps</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://scrapeops.io/nodejs-web-scraping-playbook/nodejs-superagent-fake-user-agents/"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:docusaurus_tag" content="default"><meta data-rh="true" name="robots" content="max-image-preview:large"><meta data-rh="true" property="og:title" content="NodeJs SuperAgent - Setting Fake User-Agents | ScrapeOps"><meta data-rh="true" name="description" content="How to use fake user-agents with NodeJs SuperAgent Library to avoid your scrapers from getting blocked."><meta data-rh="true" property="og:description" content="How to use fake user-agents with NodeJs SuperAgent Library to avoid your scrapers from getting blocked."><link data-rh="true" rel="icon" href="/img/favicon.svg"><link data-rh="true" rel="canonical" href="https://scrapeops.io/nodejs-web-scraping-playbook/nodejs-superagent-fake-user-agents/"><link data-rh="true" rel="alternate" href="https://scrapeops.io/nodejs-web-scraping-playbook/nodejs-superagent-fake-user-agents/" hreflang="en"><link data-rh="true" rel="alternate" href="https://scrapeops.io/nodejs-web-scraping-playbook/nodejs-superagent-fake-user-agents/" hreflang="x-default"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="ScrapeOps RSS Feed"> <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="ScrapeOps Atom Feed"> <link rel="preconnect" href="https://www.google-analytics.com"> <link rel="preconnect" href="https://www.googletagmanager.com"> <script async src="https://www.googletagmanager.com/gtag/js?id=G-QJSW9S9YH4"></script> <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-QJSW9S9YH4",{anonymize_ip:!0})</script> <script src="/get-refferer.js"></script> <script src="https://cdn.firstpromoter.com/fpr.js" async></script><link rel="stylesheet" href="/assets/css/styles.0a6705c8.css"> <script src="/assets/js/runtime~main.8a89d22b.js" defer="defer"></script> <script src="/assets/js/main.2a878338.js" defer="defer"></script> </head> <body class="navigation-with-keyboard"> <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.setAttribute(n,e)}}catch(t){}}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"><div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_s0pr" style="background-color:#0d53d7;color:#fff" role="banner"><div class="announcementBarContent_dpRF">Need a proxy solution? Try ScrapeOps and get <a target="_self" href="https://scrapeops.io/app/register/proxy/">1,000 free requests here</a>, or compare all proxy providers <a target="_self" href="https://scrapeops.io/proxy-providers/comparison/">here</a>!</div></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle clean-btn" type="button" tabindex="0"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/scrapeops-logo.svg" alt="ScrapeOps Logo" class="themedComponent_mlkZ themedComponent--light_NVdE" height="24px" width="18px"><img src="/img/scrapeops-logo.svg" alt="ScrapeOps Logo" class="themedComponent_mlkZ themedComponent--dark_xIcU" height="24px" width="18px"></div>ScrapeOps</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Solutions</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/proxy-aggregator/">Proxy Aggregator</a></li><li><a class="dropdown__link" href="/monitoring-scheduling/">Monitoring & Scheduler</a></li></ul></div><a class="navbar__item navbar__link" href="/docs/intro/">Docs</a><a href="https://scrapeops.io/proxy-providers/comparison/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Proxy Comparison<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Guides</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/web-scraping-playbook/">Web Scraping Playbook</a></li><li><a class="dropdown__link" href="/python-web-scraping-playbook/">Python Web Scraping Playbook</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/nodejs-web-scraping-playbook/">NodeJs Web Scraping Playbook</a></li><li><a class="dropdown__link" href="/python-scrapy-playbook/">Python Scrapy Playbook</a></li><li><a class="dropdown__link" href="/selenium-web-scraping-playbook/">Selenium Web Scraping Playbook</a></li><li><a class="dropdown__link" href="/puppeteer-web-scraping-playbook/">Puppeteer Web Scraping Playbook</a></li><li><a class="dropdown__link" href="/playwright-web-scraping-playbook/">Playwright Web Scraping Playbook</a></li></ul></div><a href="https://scrapeops.io/app/login/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Login<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><a href="https://scrapeops.io/app/register/main/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Signup<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div class="main-wrapper mdx-wrapper"><script type="application/ld+json">{}</script><main class="container container--fluid margin-vert--lg"><div class="row mdxPageWrapper_yV3F"><div class="col col--8"><img src="https://res.cloudinary.com/dyaskan9k/image/fetch/f_auto,q_auto/https://scrapeops-assets-2.nyc3.cdn.digitaloceanspaces.com/Playbooks/NodeJs-Web-Scraping-Playbook/Thumbnails/nodejs-superagent-fake-user-agent.png" alt="NodeJs SuperAgent: Setting Fake User-Agents" fetchpriority="high" loading="eager" class="top-header-img img_CujE"> <h1>NodeJs SuperAgent: Setting Fake User-Agents</h1> <div class="theme-admonition theme-admonition-tip admonition_xJq3 alert alert--success"><div class="admonitionHeading_Gvgb"><svg viewBox="0 0 12 16"><path fill-rule="evenodd" d="M6.5 0C3.48 0 1 2.19 1 5c0 .92.55 2.25 1 3 1.34 2.25 1.78 2.78 2 4v1h5v-1c.22-1.22.66-1.75 2-4 .45-.75 1-2.08 1-3 0-2.81-2.48-5-5.5-5zm3.64 7.48c-.25.44-.47.8-.67 1.11-.86 1.41-1.25 2.06-1.45 3.23-.02.05-.02.11-.02.17H5c0-.06 0-.13-.02-.17-.2-1.17-.59-1.83-1.45-3.23-.2-.31-.42-.67-.67-1.11C2.44 6.78 2 5.65 2 5c0-2.2 2.02-4 4.5-4 1.22 0 2.36.42 3.22 1.19C10.55 2.94 11 3.94 11 5c0 .66-.44 1.78-.86 2.48zM4 14h5c-.23 1.14-1.3 2-2.5 2s-2.27-.86-2.5-2z"></path></svg>Using Fake User-Agents With NodeJs SuperAgent Library</div><div class="admonitionContent_BuS1">To use fake user-agents with <a href="https://github.com/ladjs/superagent" target="_blank" rel="noopener noreferrer">NodeJS SuperAgent</a>, you first need to define a user-agent in a <code>headers</code> object. Then you simply call <code>set</code> method on the request with the <code>headers</code> object.<div class="language-javascript codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-javascript codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> const request = require('superagent'); (async () => { const url = 'http://httpbin.org/headers'; const headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36' } try{ const response = await request.get(url) .set(headers) console.log(response.body); } catch (error){ console.log('error', error); } })(); </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div></div></div> One of the most common reasons for getting blocked whilst web scraping is using bad user-agents. However, integrating fake user-agents into your NodeJs web scrapers that use <a href="https://github.com/ladjs/superagent" target="_blank" rel="noopener noreferrer">SuperAgent</a> is very easy. So in this guide, we will go through: <ul> <li><a href="#what-are-fake-user-agents">What Are Fake User-Agents?</a></li> <li><a href="#how-to-set-a-fake-user-agent-in-nodejs-superagent">How To Set A User Agent In NodeJs SuperAgent</a></li> <li><a href="#how-to-rotate-user-agents">How To Rotate User-Agents</a></li> <li><a href="#how-to-manage-thousands-of-fake-user-agents">How To Manage Thousands of Fake User-Agents</a></li> <li><a href="#why-use-fake-browser-headers">Why Use Fake Browser Headers</a></li> <li><a href="#scrapeops-fake-browser-headers-api">ScrapeOps Fake Browser Headers API</a></li> </ul> First, let's quickly go over some the very basics.   <div style="background-color:#f6fafe;border-radius:2px;border:1px solid #f0f0f3;padding:1.5rem;padding-bottom:2.5rem;padding-top:2.5rem;text-align:center;margin-top:2rem"><h3>Need help scraping the web?</h3>Then check out <a href="https://scrapeops.io/?utm_campaign=nodejs-web-scraping-playbook&utm_source=article&utm_medium=organic&utm_content=nodejs-superagent-fake-user-agents" style="color:#000;text-decoration:none;border-bottom:2px dashed #0d53d7;font-weight:600">ScrapeOps</a>, the complete toolkit for web scraping.<div style="display:flex;flex-direction:row;flex-wrap:wrap;justify-content:center"><div><div><img style="height:80px;width:80px" src="https://assets-scrapeops.nyc3.digitaloceanspaces.com/Icons/scrapeops-proxy-aggregator-icon.svg" alt="ScrapeOps Proxy Manager"></div><div><a href="https://scrapeops.io/proxy-aggregator/?utm_campaign=nodejs-web-scraping-playbook&utm_source=article&utm_medium=organic&utm_content=nodejs-superagent-fake-user-agents" style="color:#000;text-decoration:none;font-weight:600">Proxy Manager</a></div></div><div style="margin-left:3.5rem;margin-right:3.5rem"><div><img style="height:80px;width:80px" src="https://assets-scrapeops.nyc3.digitaloceanspaces.com/Icons/scrapeops-monitoring-icon.svg" alt="ScrapeOps Monitoring"></div><div><a href="https://scrapeops.io/monitoring-scheduling/?utm_campaign=nodejs-web-scraping-playbook&utm_source=article&utm_medium=organic&utm_content=nodejs-superagent-fake-user-agents" style="color:#000;text-decoration:none;font-weight:600">Scraper Monitoring</a></div></div><div><div><img style="height:80px;width:80px" src="https://assets-scrapeops.nyc3.digitaloceanspaces.com/Icons/scrapeops-scheduler-icon.svg" alt="ScrapeOps Job Scheduling"></div><div><a href="https://scrapeops.io/monitoring-scheduling?utm_campaign=nodejs-web-scraping-playbook&utm_source=article&utm_medium=organic&utm_content=nodejs-superagent-fake-user-agents" style="color:#000;text-decoration:none;font-weight:600">Job Scheduling</a></div></div></div></div> <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-are-fake-user-agents">What Are Fake User-Agents?<a href="#what-are-fake-user-agents" class="hash-link" aria-label="Direct link to What Are Fake User-Agents?" title="Direct link to What Are Fake User-Agents?"></a></h2> User Agents are strings that let the website you are scraping identify the application, operating system (OSX/Windows/Linux), browser (Chrome/Firefox/Internet Explorer), etc. of the user sending a request to their website. They are sent to the server as part of the request headers. Here is an example User agent sent when you visit a website with a Chrome browser: <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36' </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> When scraping a website, you also need to set user-agents on every request as otherwise the website may block your requests because it knows you aren't a real user. In the case of <a href="https://github.com/ladjs/superagent" target="_blank" rel="noopener noreferrer">SuperAgent</a>, when you send a request by default it doesn't include a User-Agent. This will clearly identify that the request isn't being made with a real browser. <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> 'User-Agent': '', </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> This user-agent will clearly identify your requests as suspicious, so the website can easily block you from scraping the site. That is why we need to manage the user-agents we use with SuperAgent when we send requests. <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="how-to-set-a-fake-user-agent-in-nodejs-superagent">How To Set A Fake User-Agent In NodeJs SuperAgent<a href="#how-to-set-a-fake-user-agent-in-nodejs-superagent" class="hash-link" aria-label="Direct link to How To Set A Fake User-Agent In NodeJs SuperAgent" title="Direct link to How To Set A Fake User-Agent In NodeJs SuperAgent"></a></h2> Setting <a href="https://ladjs.github.io/superagent/" target="_blank" rel="noopener noreferrer">SuperAgent</a> to use a fake user-agent is very easy. You just need to define a user-agent in a <code>headers</code> object and pass it during the <code>set</code> method call. <div class="language-javascript codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-javascript codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> const request = require('superagent'); (async () => { const url = 'http://httpbin.org/headers'; const headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36' } try{ const response = await request.get(url) .set(headers) console.log(response.body); } catch (error){ console.log('error', error); } })(); </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> From here SuperAgent will use the above user-agent to make the request. All request headers, including User-Agent, are reflected in response JSON, which can be accessed with <code>response.body</code>. <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="how-to-rotate-user-agents">How To Rotate User-Agents<a href="#how-to-rotate-user-agents" class="hash-link" aria-label="Direct link to How To Rotate User-Agents" title="Direct link to How To Rotate User-Agents"></a></h2> Rotating through user-agents is also pretty straightforward when using NodeJS SuperAgent library. We just need a list of user-agents in our scraper and use a random one with every request. <div class="language-javascript codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-javascript codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> const request = require('superagent'); const randomUserAgent = require('random-useragent'); const userAgents = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36', 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_4_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1', 'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363' ]; const url = 'http://httpbin.org/headers' const headers = { 'User-Agent': randomUserAgent.getRandom(userAgents) }; request.get(url) .set(headers) .then(async response => { console.log(await response.body); }) .catch(error => { console.error(error); }); </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> This works but it has drawbacks as we would need to build & keep an up-to-date list of user-agents ourselves. <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="how-to-manage-thousands-of-fake-user-agents">How To Manage Thousands of Fake User-Agents<a href="#how-to-manage-thousands-of-fake-user-agents" class="hash-link" aria-label="Direct link to How To Manage Thousands of Fake User-Agents" title="Direct link to How To Manage Thousands of Fake User-Agents"></a></h2> A better approach would be to use a free user-agent API like <a href="/docs/fake-user-agent-headers-api/overview/">ScrapeOps Fake User-Agent API</a> to download an up-to-date user-agent list when your scraper starts up and then pick a random user-agent for each request. To use the ScrapeOps Fake User-Agents API you just need to send a request to the API endpoint to retrieve a list of user-agents. <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> http://headers.scrapeops.io/v1/user-agents?api_key=YOUR_API_KEY </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> To use the ScrapeOps Fake User-Agent API, you first need an API key which you can get by signing up for a <a href="https://scrapeops.io/app/register/headers/" target="_blank" rel="noopener noreferrer">free account here</a>. Example response from the API: <div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> { "result": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Windows; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8", "Mozilla/5.0 (Windows NT 10.0; Windows; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Windows; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36" ] } </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> To integrate the Fake User-Agent API you should configure your scraper to retrieve a batch of the most up-to-date user-agents when the scraper starts and then configure your scraper to pick a random user-agent from this list for each request. Here is an example NodeJs SuperAgent scraper integration: <div class="language-javascript codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-javascript codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> const request = require('superagent'); const SCRAPEOPS_API_KEY = 'YOUR_API_KEY'; const url = `http://headers.scrapeops.io/v1/user-agents?api_key=${SCRAPEOPS_API_KEY}` async function getUserAgentList() { const response = await request.get(url); const data = response.body return data.result || []; } function getRandomUserAgent(userAgentList) { const randomIndex = Math.floor(Math.random() * userAgentList.length); return userAgentList[randomIndex]; } const urlList = [ 'https://jsonplaceholder.typicode.com/posts/1', 'https://jsonplaceholder.typicode.com/posts/2', 'https://jsonplaceholder.typicode.com/posts/3' ]; (async () => { try { const userAgentList = await getUserAgentList(); for (const url of urlList) { const headers = { 'User-Agent': getRandomUserAgent(userAgentList) }; const response = await request.get(url) .set(headers); console.log(response.body); } } catch (error) { console.error(error); } })(); </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> Here the scraper will use a random user-agent for each request. <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-use-fake-browser-headers">Why Use Fake Browser Headers<a href="#why-use-fake-browser-headers" class="hash-link" aria-label="Direct link to Why Use Fake Browser Headers" title="Direct link to Why Use Fake Browser Headers"></a></h2> For simple websites, simply setting an up-to-date user-agent should allow you to scrape a website pretty reliably. However, a lot of popular websites are increasingly using sophisticated anti-bot technologies to try and prevent developer from scraping data from their websites. These anti-bot solutions not only look at your requests user-agent when analysing the request, but also the other headers a real browser normally sends. By using a full set of browser headers you make your requests look more like real user requests, and as a result harder to detect. Here are example headers when using a Chrome browser on a MacOS machine: <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv">sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99" sec-ch-ua-mobile: ?0 sec-ch-ua-platform: "macOS" Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 Sec-Fetch-Site: none Sec-Fetch-Mode: navigate Sec-Fetch-User: ?1 Sec-Fetch-Dest: document Accept-Encoding: gzip, deflate, br Accept-Language: en-GB,en-US;q=0.9,en;q=0.8 </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> As we can see, real browsers don't just send <code>User-Agent</code> strings but also a number of other headers that are used to identify and customize the request. So to improve the reliability of our scrapers we should also include these headers when making requests. You could build a list of fake browser headers yourself, or you could use the <a href="/docs/fake-user-agent-headers-api/fake-browser-headers/">ScrapeOps Fake Browser Headers API</a> to get an up-to-date list every time your scraper starts up. <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="scrapeops-fake-browser-headers-api">ScrapeOps Fake Browser Headers API<a href="#scrapeops-fake-browser-headers-api" class="hash-link" aria-label="Direct link to ScrapeOps Fake Browser Headers API" title="Direct link to ScrapeOps Fake Browser Headers API"></a></h2> The <a href="/docs/fake-user-agent-headers-api/fake-browser-headers/">ScrapeOps Fake Browser Headers API</a> is a free API that returns a list of optimized fake browser headers that you can use in your web scrapers to avoid blocks/bans and improve the reliability of your scrapers. API Endpoint: <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> http://headers.scrapeops.io/v1/browser-headers?api_key=YOUR_API_KEY </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> Response: <div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> { "result": [ { "upgrade-insecure-requests": "1", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Windows; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "sec-ch-ua": "\".Not/A)Brand\";v=\"99\", \"Google Chrome\";v=\"103\", \"Chromium\";v=\"103\"", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\"Windows\"", "sec-fetch-site": "none", "sec-fetch-mod": "", "sec-fetch-user": "?1", "accept-encoding": "gzip, deflate, br", "accept-language": "bg-BG,bg;q=0.9,en-US;q=0.8,en;q=0.7" }, { "upgrade-insecure-requests": "1", "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "sec-ch-ua": "\".Not/A)Brand\";v=\"99\", \"Google Chrome\";v=\"103\", \"Chromium\";v=\"103\"", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\"Linux\"", "sec-fetch-site": "none", "sec-fetch-mod": "", "sec-fetch-user": "?1", "accept-encoding": "gzip, deflate, br", "accept-language": "fr-CH,fr;q=0.9,en-US;q=0.8,en;q=0.7" } ] } </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> To use the ScrapeOps Fake Browser Headers API, you first need an API key which you can get by signing up for a <a href="https://scrapeops.io/app/register/headers/" target="_blank" rel="noopener noreferrer">free account here</a>. To integrate the Fake Browser Headers API you should configure your scraper to retrieve a batch of the most up-to-date headers when the scraper starts and then configure your scraper to pick a random header from this list for each request. Here is an example NodeJs SuperAgent scraper integration: <div class="language-javascript codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#d6deeb;--prism-background-color:#011627"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-javascript codeBlock_bY9V thin-scrollbar" style="color:#d6deeb;background-color:#011627"><code class="codeBlockLines_e6Vv"> const request = require('superagent'); const SCRAPEOPS_API_KEY = 'YOUR_API_KEY'; const url = `http://headers.scrapeops.io/v1/browser-headers?api_key=${SCRAPEOPS_API_KEY}`; async function getHeadersList() { const response = await request.get(url); return response.body.result || []; } function getRandomHeader(headerList) { const randomIndex = Math.floor(Math.random() * headerList.length); return headerList[randomIndex]; } const urlList = [ 'https://jsonplaceholder.typicode.com/posts/1', 'https://jsonplaceholder.typicode.com/posts/2', 'https://jsonplaceholder.typicode.com/posts/3' ]; (async () => { try { const headerList = await getHeadersList(); for (const url of urlList) { const headers = getRandomHeader(headerList); const response = await request.get(url) .set(headers); console.log(response.body); } } catch (error) { console.error(error); } })(); </code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></button></div></div></div> For more information on check out the <a href="/docs/fake-user-agent-headers-api/fake-browser-headers/">Fake Browser Headers API documenation</a>. <hr> <h2 class="anchor anchorWithStickyNavbar_LWe7" id="more-web-scraping-tutorials">More Web Scraping Tutorials<a href="#more-web-scraping-tutorials" class="hash-link" aria-label="Direct link to More Web Scraping Tutorials" title="Direct link to More Web Scraping Tutorials"></a></h2> So that's how you can set fake user-agents when scraping with NodeJs SuperAgent. If you would like to learn more about Web Scraping, then be sure to check out <a href="/web-scraping-playbook/">The Web Scraping Playbook</a>. Or check out one of our more in-depth guides: <ul> <li><a href="/web-scraping-playbook/web-scraping-without-getting-blocked/">How to Scrape The Web Without Getting Blocked Guide</a></li> <li><a href="https://scrapeops.io/blog/the-state-of-web-scraping-2022/" target="_blank" rel="noopener noreferrer">The State of Web Scraping 2020</a></li> <li><a href="/web-scraping-playbook/ethics-of-web-scraping/">The Ethics of Web Scraping</a></li> </ul></div><div class="col col--2"><div class="tableOfContents_bqdL thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-are-fake-user-agents" class="table-of-contents__link toc-highlight">What Are Fake User-Agents?</a></li><li><a href="#how-to-set-a-fake-user-agent-in-nodejs-superagent" class="table-of-contents__link toc-highlight">How To Set A Fake User-Agent In NodeJs SuperAgent</a></li><li><a href="#how-to-rotate-user-agents" class="table-of-contents__link toc-highlight">How To Rotate User-Agents</a></li><li><a href="#how-to-manage-thousands-of-fake-user-agents" class="table-of-contents__link toc-highlight">How To Manage Thousands of Fake User-Agents</a></li><li><a href="#why-use-fake-browser-headers" class="table-of-contents__link toc-highlight">Why Use Fake Browser Headers</a></li><li><a href="#scrapeops-fake-browser-headers-api" class="table-of-contents__link toc-highlight">ScrapeOps Fake Browser Headers API</a></li><li><a href="#more-web-scraping-tutorials" class="table-of-contents__link toc-highlight">More Web Scraping Tutorials</a></li></ul></div></div></div></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Resources</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/intro/">Documentation</a></li><li class="footer__item"><a href="https://scrapeops.io/proxy-providers/comparison/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Proxy Comparison Tool</a></li><li class="footer__item"><a class="footer__link-item" href="/blog/">Blog</a></li><li class="footer__item"><a href="https://github.com/ScrapeOps" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub</a></li></ul></div><div class="col footer__col"><div class="footer__title">Web Scraping Guides</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/web-scraping-playbook/">Web Scraping Playbook</a></li><li class="footer__item"><a class="footer__link-item" href="/python-web-scraping-playbook/">Python Web Scraping Playbook</a></li><li class="footer__item"><a class="footer__link-item" href="/nodejs-web-scraping-playbook/">NodeJs Web Scraping Playbook</a></li><li class="footer__item"><a class="footer__link-item" href="/python-scrapy-playbook/">Python Scrapy Playbook</a></li><li class="footer__item"><a class="footer__link-item" href="/selenium-web-scraping-playbook/">Selenium Web Scraping Playbook</a></li><li class="footer__item"><a class="footer__link-item" href="/puppeteer-web-scraping-playbook/">Puppeteer Web Scraping Playbook</a></li><li class="footer__item"><a class="footer__link-item" href="/playwright-web-scraping-playbook/">Playwright Web Scraping Playbook</a></li></ul></div><div class="col footer__col"><div class="footer__title">Company</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/affiliate-program/">Affiliate Program</a></li><li class="footer__item"><a class="footer__link-item" href="/privacy-policy/">Privacy Policy</a></li><li class="footer__item"><a class="footer__link-item" href="/terms-of-service/">Terms Of Service</a></li><li class="footer__item"><a class="footer__link-item" href="/data-protection-policy/">Data Protection Policy</a></li><li class="footer__item"><a class="footer__link-item" href="/data-processing-agreement/">Data Processing Agreement</a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 ScrapeOps.</div></div></div></footer></div> </body> </html>