Alexa's Web and Site Audit Crawlers

<!DOCTYPE html> <html dir="ltr" lang="en-US"> <head><script type="text/javascript" src="/_static/js/bundle-playback.js?v=HxkREWBo" charset="utf-8"></script> <script type="text/javascript" src="/_static/js/wombat.js?v=txqj7nKC" charset="utf-8"></script> <script>window.RufflePlayer=window.RufflePlayer||{};window.RufflePlayer.config={"autoplay":"on","unmuteOverlay":"hidden"};</script> <script type="text/javascript" src="/_static/js/ruffle/ruffle.js"></script> <script type="text/javascript"> __wm.init("https://web.archive.org/web"); __wm.wombat("https://support.alexa.com/hc/en-us/articles/200450194","20181114080059","https://web.archive.org/","web","/_static/", "1542182459"); </script> <link rel="stylesheet" type="text/css" href="/_static/css/banner-styles.css?v=S1zqJCYt" /> <link rel="stylesheet" type="text/css" href="/_static/css/iconochive.css?v=3PDvdIFv" />  <meta charset="utf-8"/>  <title>Alexa's Web and Site Audit Crawlers – Alexa Support</title> <meta name="csrf-param" content="authenticity_token"/> <meta name="csrf-token" content="YlOolJNKAIiTrkQMUnFEHR0hzT4k9in8pRQ1t+On4UNVDb9ZhRB7gd5YdvnsexNtiBw0X8sESo4n3TKL4nQAPQ=="/> <meta name="description" content="Information regarding our privacy policy, company, and technology can be found on the following pages Privacy Policy and About Us. Alexa..."/><meta property="og:image" content="https://web.archive.org/web/20181114080059im_/https://theme.zdassets.com/theme_assets/53743/4ee12b36beba72e0cc0d4056d74ce26536364d8b.png"/> <meta property="og:type" content="website"/> <meta property="og:site_name" content="Alexa Support"/> <meta property="og:title" content="Alexa's Web and Site Audit Crawlers"/> <meta property="og:description" content="Information regarding our privacy policy, company, and technology can be found on the following pages Privacy Policy and About Us. Alexa crawls the web in order to identify and classify web conten..."/> <meta property="og:url" content="https://web.archive.org/web/20181114080059/http://support.alexa.com/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers"/> <link rel="canonical" href="https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers"/> <link rel="alternate" hreflang="en" href="https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers"/>  <link rel="stylesheet" media="all" href="//web.archive.org/web/20181114080059cs_/https://static.zdassets.com/hc/assets/application-6229f1ebd2f90b23df13e8c12bfd8ab1.css" id="stylesheet"/> <link rel="stylesheet" type="text/css" href="//web.archive.org/web/20181114080059cs_/https://p14.zdassets.com/hc/theming_assets/53743/411634/style.css?digest=360156022794"/> <link rel="shortcut icon" type="image/x-icon" href="//web.archive.org/web/20181114080059im_/https://theme.zdassets.com/theme_assets/53743/077c8a58ad9cce4cfec0a54c2846bb390d8987e8.ico"/>  <script src="//web.archive.org/web/20181114080059js_/https://static.zdassets.com/hc/assets/jquery-b60ddb79ff2563b75442a6bac88b00b5.js"></script> <script> (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','//web.archive.org/web/20181114080059/https://www.google-analytics.com/analytics.js','ga'); ga('create', 'UA-48920324-1', 'auto'); ga('send', 'pageview'); </script> <meta name="viewport" content="width=device-width, initial-scale=1.0"/> <link href="https://web.archive.org/web/20181114080059cs_/https://fonts.googleapis.com/css?family=Roboto:100,100i,300,300i,400,400i,500,500i,700,700i,900,900i" rel="stylesheet"> <link rel="stylesheet" href="//web.archive.org/web/20181114080059cs_/https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"/> <link rel="stylesheet" href="https://web.archive.org/web/20181114080059cs_/https://use.fontawesome.com/releases/v5.3.1/css/all.css" integrity="" crossorigin="anonymous"> <link rel="stylesheet" href="//web.archive.org/web/20181114080059cs_/https://cdn.jsdelivr.net/highlight.js/9.10.0/styles/github.min.css"/> <link rel="stylesheet" href="//web.archive.org/web/20181114080059cs_/https://cdn.jsdelivr.net/jquery.magnific-popup/1.0.0/magnific-popup.css"/> <script src="//web.archive.org/web/20181114080059js_/https://cdnjs.cloudflare.com/ajax/libs/magnific-popup.js/1.1.0/jquery.magnific-popup.min.js"></script> <script src="//web.archive.org/web/20181114080059js_/https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.10.0/highlight.min.js"></script> <script src="//web.archive.org/web/20181114080059js_/https://theme.zdassets.com/theme_assets/53743/746fe88622e170aa9b416e1495a8654c888ca107.js"></script> <script src="//web.archive.org/web/20181114080059js_/https://cdnjs.cloudflare.com/ajax/libs/store.js/1.3.20/store.min.js"></script> <script src="//web.archive.org/web/20181114080059js_/https://cdnjs.cloudflare.com/ajax/libs/lz-string/1.4.4/lz-string.min.js"></script> <script src="//web.archive.org/web/20181114080059js_/https://theme.zdassets.com/theme_assets/53743/b12a0a4d805007175080ec86dcb96c30dd752841.js"></script> <script src="//web.archive.org/web/20181114080059js_/https://theme.zdassets.com/theme_assets/53743/82636578ca70b58d35d9f8fd3a808364da91e1ce.js"></script> <script type="text/javascript" src="//web.archive.org/web/20181114080059js_/https://p14.zdassets.com/hc/theming_assets/53743/411634/script.js?digest=360156022794"></script> </head> <body class=""> <div class="js-alert-box"></div> <div class="layout"> <header class="topbar container" data-topbar> <div class="container-inner"> <div class="topbar__inner"> <div class="topbar__col clearfix"> <div class="logo-wrapper"> <div class="logo"> <a title="Home" href="/web/20181114080059/https://support.alexa.com/hc/en-us"> <img src="//web.archive.org/web/20181114080059im_/https://theme.zdassets.com/theme_assets/53743/4ee12b36beba72e0cc0d4056d74ce26536364d8b.png" alt="Logo"> </a> </div> </div> <button type="button" role="button" aria-label="Toggle Navigation" class="lines-button x" data-toggle-menu> <span class="lines"></span> </button> </div> <div class="topbar__col topbar__menu"> <div class="topbar__collapse" data-menu> <div class="topbar__controls"> <a class="btn btn--topbar submit-a-request" href="/web/20181114080059/https://support.alexa.com/hc/en-us/requests/new">Submit a request</a> <a class="login" data-auth-action="signin" role="button" href="/web/20181114080059/https://support.alexa.com/hc/en-us/signin?return_to=https%3A%2F%2Fsupport.alexa.com%2Fhc%2Fen-us%2Farticles%2F200450194&locale=1">Sign in</a> </div> </div> </div> </div> </div> </header> <main role="main"> <div class="hero-unit container" data-hero-unit> <div class="container-inner"> <div class="intro-title">Questions? We have answers.</div> <div class="hero-unit__search"> <form role="search" class="search" data-search="" data-instant="true" autocomplete="off" action="/web/20181114080059/https://support.alexa.com/hc/en-us/search" accept-charset="UTF-8" method="get"><input name="utf8" type="hidden" value="✓"/><input type="search" name="query" id="query" placeholder="Search" autocomplete="off" aria-label="Search"/> <input type="submit" name="commit" value="Search"/></form> </div> </div> </div> <div class="container" data-article-page> <div class="container-inner"> <div class="row clearfix"> <div class="column column--sm-12 center-block"> <ol class="breadcrumbs"> <li title="Alexa Support"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us">Alexa Support</a> </li> <li title="General"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/categories/200014214-General">General</a> </li> <li title="Crawlers"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/sections/200100794-Crawlers">Crawlers</a> </li> </ol> <article class="article clearfix" itemscope itemtype="http://schema.org/Article"> <header class="article-header"> <h1 class="article__title" itemprop="name"> Alexa's Web and Site Audit Crawlers </h1>  </header> <div class="article-body markdown" itemprop="articleBody"> <p>Information regarding our privacy policy, company, and technology can be found on the following pages <a href="https://web.archive.org/web/20181114080059/http://www.alexa.com/help/privacy">Privacy Policy</a> and <a href="https://web.archive.org/web/20181114080059/http://www.alexa.com/about">About Us</a>.</p> <p>Alexa crawls the web in order to identify and classify web content, to discover backlinks, and find related sites. We will not index anything you would like to remain private.</p> <p><strong>What if I don't want Alexa to crawl my site?</strong></p> <p>All you have to do is tell us using a <em>robots.txt</em> file.</p> <p>Robots.txt is a file website administrators can place at the top level of a site to direct the behavior of web crawling robots. All of the major Web-crawlers such as Google, Yahoo, Bing and Baidu respect robots.txt.</p> <p>The Alexa web crawler (robot) identifies itself as “ia_archiver” in the HTTP “User-agent” header field. The Alexa Internet ia_archiver crawler strictly adheres to robots.txt rules.</p> <p>To <strong>prevent</strong> ia_archiver from visiting any part of your site, your robots.txt file should look like this:</p> <p>User-agent: ia_archiver<br>Disallow: /</p> <p>You can also restrict crawling of specific directories. For example, to prevent ia_archiver from visiting the images directory (and its subdirectories):</p> <p>User-agent: ia_archiver<br>Disallow: /images/</p> <p>To <strong>allow</strong> ia_archiver to visit your entire site add these lines to your robots.txt file:</p> <p>User-agent: ia_archiver<br>Disallow:</p> <p>For more information regarding robots, crawling, and robots.txt visit the Web Robots Pages at <a href="https://web.archive.org/web/20181114080059/http://www.robotstxt.org/">Robotstxt.org</a>, an excellent source for the latest information on the Standard for Robots Exclusion.</p> </div> </article>  <div class="article-more-questions">Have more questions? <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/requests/new">Submit a request</a></div> <hr> <section class="section-articles"> <h4 class="section-articles__title">Articles in this section</h4> <ul class="section-articles__list"> <li class="section-articles__item"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/articles/200450254-Why-is-Alexa-crawling-my-site-" class="section-articles__link ">Why is Alexa crawling my site?</a> </li> <li class="section-articles__item"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers" class="section-articles__link is-active">Alexa's Web and Site Audit Crawlers</a> </li> <li class="section-articles__item"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/articles/200462340-Certification-Crawler-Information" class="section-articles__link ">Certification Crawler Information</a> </li> <li class="section-articles__item"> <a href="/web/20181114080059/https://support.alexa.com/hc/en-us/articles/200384195-How-do-Alexa-s-crawlers-identify-themselves-" class="section-articles__link ">How do Alexa’s crawlers identify themselves?</a> </li> </ul> </section> </div> </div> </main> </div>  <footer class="footer container"> <div class="container-inner footer__inner"> <div class="footer__col copyright"> <p> © Alexa Internet, Inc. 1996 - 2018 | <a href="https://web.archive.org/web/20181114080059/https://www.alexa.com/help/privacy">Privacy</a> | <a href="https://web.archive.org/web/20181114080059/https://www.alexa.com/help/terms">Terms</a> | <a href="https://web.archive.org/web/20181114080059/https://www.alexa.com/contact-us">Contact</a> | <a href="https://web.archive.org/web/20181114080059/https://aws.amazon.com/alexa/">API</a> </p> </div> <div class="footer__col footer__col--social-links"> <a href="https://web.archive.org/web/20181114080059/https://www.alexa.com/" target="_blank" class="footer-social-link fa-lg"> <img src="//web.archive.org/web/20181114080059im_/https://theme.zdassets.com/theme_assets/53743/5e463e7a5e20f2d2351bf4242ec7b2ecad4db661.png" alt="alexa.com"> </a> <a href="https://web.archive.org/web/20181114080059/https://blog.alexa.com/" target="_blank" class="footer-social-link fas fa-newspaper-o fa-lg"></a> <a href="https://web.archive.org/web/20181114080059/https://www.facebook.com/alexainternet" target="_blank" class="footer-social-link fab fa-facebook-f fa-lg"></a> <a href="https://web.archive.org/web/20181114080059/https://twitter.com/Alexa_Support" target="_blank" class="footer-social-link fab fa-twitter fa-lg"></a> <a href="https://web.archive.org/web/20181114080059/http://www.linkedin.com/company/alexa-internet" target="_blank" class="footer-social-link fab fa-linkedin-in fa-lg"></a> </div> </div> <script type="text/javascript" src="https://web.archive.org/web/20181114080059js_/https://cdn.solvvy.com/deflect/customization/alexa/solvvy.js" async></script> </footer> <a href="#" class="scroll-to-top fa fa-angle-up" data-scroll-to-top></a>  <script type="text/javascript" src="//web.archive.org/web/20181114080059js_/https://static.zdassets.com/hc/assets/locales/en-us-3d4abef49badc2dd07cfd08d524a66c3.js"></script> <script src="https://web.archive.org/web/20181114080059js_/https://alexa.zendesk.com/auth/v2/host.js" data-brand-id="411634" data-return-to="https://support.alexa.com/hc/en-us/articles/200450194" data-theme="hc" data-locale="1" data-auth-origin="411634,true,true"></script> <script type="text/javascript"> /* Greetings sourcecode lurker! This is for internal Zendesk and legacy usage, we don't support or guarantee any of these values so please don't build stuff on top of them. */ HelpCenter = {}; HelpCenter.account = {"subdomain":"alexa","environment":"production","name":"Alexa Support"}; HelpCenter.user = {"identifier":"da39a3ee5e6b4b0d3255bfef95601890afd80709","email":null,"name":null,"role":"anonymous","avatar_url":"https://web.archive.org/web/20181114080059/https://assets.zendesk.com/hc/assets/default_avatar.png","is_admin":false,"organizations":[],"groups":[]}; HelpCenter.internal = {"asset_url":"//web.archive.org/web/20181114080059/https://static.zdassets.com/hc/assets/","web_widget_asset_composer_url":"https://web.archive.org/web/20181114080059/https://static.zdassets.com/ekr/snippet.js","current_session":{"locale":"en-us","csrf_token":"vvNfY4thIWqutJTXBUTzBXCrfA9NirENoxT3/yXEXEWJrUiunTtaY+NCpiK7TqR15ZaFbqJ40n8h3fDDJBe9Ow==","shared_csrf_token":null},"settings":{"zopim_enabled":false,"spam_filter_enabled":true},"current_record_id":"200450194","current_record_url":"/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers","current_record_title":"Alexa's Web and Site Audit Crawlers","search_results_count":null,"current_text_direction":"ltr","current_brand_url":"https://web.archive.org/web/20181114080059/https://alexa.zendesk.com","current_host_mapping":"support.alexa.com","current_path":"/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers","authentication_domain":"https://web.archive.org/web/20181114080059/https://alexa.zendesk.com","show_autocomplete_breadcrumbs":true,"rollbar_config":null,"user_info_changing_enabled":false,"has_user_profiles_enabled":false,"has_anonymous_kb_voting":false,"has_professional_upsell":false,"has_multi_language_help_center":true,"mobile_device":false,"mobile_site_enabled":true,"show_at_mentions":false,"has_copied_content":false,"embeddables_config":{"embeddables_web_widget":false,"embeddables_connect_ipms":false},"base_domain":"zendesk.com","answer_bot_subdomain":"static","plans_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/admin/plan?locale=en-us","manage_content_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/en-us","arrange_content_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/admin/arrange_contents?locale=en-us","general_settings_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/admin/general_settings?locale=en-us","user_segments_url":"https://web.archive.org/web/20181114080059/https://alexa.zendesk.com/knowledge/user_segments?brand_id=411634","import_articles_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/admin/import_articles?locale=en-us","has_community_enabled":false,"has_groups":true,"has_internal_sections":true,"has_organizations":true,"has_tag_restrictions":true,"has_answer_bot_web_form_enabled":false,"has_close_the_loop_enabled":false,"billing_url":"/access/return_to?return_to=https://alexa.zendesk.com/agent/admin/subscription","has_answer_bot":true,"has_block_answer_bot_embeddable":false,"has_guide_docs_importer":true,"has_guide_permission_groups":true,"answer_bot_management_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/admin/answer_bot?locale=en-us","is_account_owner":false,"has_theming_templates":true,"theming_center_url":"https://web.archive.org/web/20181114080059/https://support.alexa.com/theming","theming_cookie_key":"hc-da39a3ee5e6b4b0d3255bfef95601890afd80709-preview","is_preview":false,"permission_groups_url":"https://web.archive.org/web/20181114080059/https://alexa.zendesk.com/knowledge/permissions?brand_id=411634","has_guide_user_segments_search":false,"has_guide_user_segments_v2":true,"has_alternate_templates":false}; </script> <script src="//web.archive.org/web/20181114080059js_/https://static.zdassets.com/hc/assets/hc_enduser-2bf0cc3d29dc97ba2105767a10689dfd.js"></script> <script type="text/javascript"> (function() { var Tracker = {}; Tracker.track = function(eventName, data) { var url = "https://web.archive.org/web/20181114080059/https://support.alexa.com/hc/tracking/events?locale=en-us"; var payload = { "event": eventName, "data": data, "referrer": document.referrer }; var xhr = new XMLHttpRequest(); xhr.open("POST", url, true); xhr.setRequestHeader("Content-Type", "application/json; charset=UTF-8"); xhr.send(JSON.stringify(payload)); }; Tracker.track("article_viewed", "BAh7CDoLbG9jYWxlSSIKZW4tdXMGOgZFVDoPYXJ0aWNsZV9pZGkEkqDyCzoKX21ldGF7CzoPYWNjb3VudF9pZGkC79E6E2hlbHBfY2VudGVyX2lkaQSM0OsLOg1icmFuZF9pZGkD8kcGOgx1c2VyX2lkMDoOdXNlcl9yb2xlSSIOYW5vbnltb3VzBjsGVDsASSIKZW4tdXMGOwZU--7ce962bc384574ca47b0b10c12df66ea6cb8dc48"); })(); </script> </body> </html>

CINXE.COM

Alexa's Web and Site Audit Crawlers – Alexa Support