CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Web crawler - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"ba972a2b-b9ae-4ba7-ae54-d6341bc5c2b8","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Web_crawler","wgTitle":"Web crawler","wgCurRevisionId":1257967634,"wgRevisionId":1257967634,"wgArticleId":33120,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Webarchive template wayback links","CS1 maint: multiple names: authors list","CS1 errors: missing periodical","Articles with short description","Short description is different from Wikidata","Use dmy dates from September 2020","All articles with unsourced statements","Articles with unsourced statements from February 2023","Articles with unsourced statements from June 2021","Search engine software","Web crawlers","Internet search algorithms"],"wgPageViewLanguage":"en", "wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Web_crawler","wgRelevantArticleId":33120,"wgIsProbablyEditable":true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":50000,"wgRelatedArticlesCompat":[],"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q45842", "wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.cite.styles":"ready","ext.math.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","mediawiki.page.media","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc", "skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession","wikibase.sidebar.tracking"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.math.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/1200px-WebCrawlerArchitecture.svg.png"> <meta property="og:image:width" content="1200"> <meta property="og:image:height" content="917"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/800px-WebCrawlerArchitecture.svg.png"> <meta property="og:image:width" content="800"> <meta property="og:image:height" content="611"> <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/640px-WebCrawlerArchitecture.svg.png"> <meta property="og:image:width" content="640"> <meta property="og:image:height" content="489"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Web crawler - Wikipedia"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Web_crawler"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Web_crawler&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Web_crawler"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Web_crawler rootpage-Web_crawler skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia"><span>Contents</span></a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events"><span>Current events</span></a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x"><span>Random article</span></a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works"><span>About Wikipedia</span></a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia"><span>Contact us</span></a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia"><span>Help</span></a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia"><span>Learn to edit</span></a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors"><span>Community portal</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia"><span>Upload file</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Web+crawler" title="You are encouraged to create an account and log in; however, it is not mandatory" class=""><span>Create account</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Web+crawler" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Web+crawler" title="You are encouraged to create an account and log in; however, it is not mandatory"><span class="vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd"></span> <span>Create account</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Web+crawler" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing"><span>learn more</span></a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y"><span>Contributions</span></a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n"><span>Talk</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-Nomenclature" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Nomenclature"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Nomenclature</span> </div> </a> <ul id="toc-Nomenclature-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Overview" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Overview"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Overview</span> </div> </a> <ul id="toc-Overview-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Crawling_policy" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Crawling_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Crawling policy</span> </div> </a> <button aria-controls="toc-Crawling_policy-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Crawling policy subsection</span> </button> <ul id="toc-Crawling_policy-sublist" class="vector-toc-list"> <li id="toc-Selection_policy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Selection_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Selection policy</span> </div> </a> <ul id="toc-Selection_policy-sublist" class="vector-toc-list"> <li id="toc-Restricting_followed_links" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Restricting_followed_links"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.1</span> <span>Restricting followed links</span> </div> </a> <ul id="toc-Restricting_followed_links-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-URL_normalization" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#URL_normalization"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.2</span> <span>URL normalization</span> </div> </a> <ul id="toc-URL_normalization-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Path-ascending_crawling" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Path-ascending_crawling"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.3</span> <span>Path-ascending crawling</span> </div> </a> <ul id="toc-Path-ascending_crawling-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Focused_crawling" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Focused_crawling"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.4</span> <span>Focused crawling</span> </div> </a> <ul id="toc-Focused_crawling-sublist" class="vector-toc-list"> <li id="toc-Academic_focused_crawler" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#Academic_focused_crawler"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.4.1</span> <span>Academic focused crawler</span> </div> </a> <ul id="toc-Academic_focused_crawler-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Semantic_focused_crawler" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#Semantic_focused_crawler"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.4.2</span> <span>Semantic focused crawler</span> </div> </a> <ul id="toc-Semantic_focused_crawler-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> </ul> </li> <li id="toc-Re-visit_policy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Re-visit_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.2</span> <span>Re-visit policy</span> </div> </a> <ul id="toc-Re-visit_policy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Politeness_policy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Politeness_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.3</span> <span>Politeness policy</span> </div> </a> <ul id="toc-Politeness_policy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Parallelization_policy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Parallelization_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.4</span> <span>Parallelization policy</span> </div> </a> <ul id="toc-Parallelization_policy-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Architectures" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Architectures"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Architectures</span> </div> </a> <ul id="toc-Architectures-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Security" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Security"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Security</span> </div> </a> <ul id="toc-Security-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Crawler_identification" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Crawler_identification"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>Crawler identification</span> </div> </a> <ul id="toc-Crawler_identification-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Crawling_the_deep_web" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Crawling_the_deep_web"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>Crawling the deep web</span> </div> </a> <ul id="toc-Crawling_the_deep_web-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Visual_vs_programmatic_crawlers" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Visual_vs_programmatic_crawlers"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>Visual vs programmatic crawlers</span> </div> </a> <ul id="toc-Visual_vs_programmatic_crawlers-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-List_of_web_crawlers" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#List_of_web_crawlers"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>List of web crawlers</span> </div> </a> <button aria-controls="toc-List_of_web_crawlers-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle List of web crawlers subsection</span> </button> <ul id="toc-List_of_web_crawlers-sublist" class="vector-toc-list"> <li id="toc-Historical_web_crawlers" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Historical_web_crawlers"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.1</span> <span>Historical web crawlers</span> </div> </a> <ul id="toc-Historical_web_crawlers-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-In-house_web_crawlers" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#In-house_web_crawlers"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.2</span> <span>In-house web crawlers</span> </div> </a> <ul id="toc-In-house_web_crawlers-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Commercial_web_crawlers" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Commercial_web_crawlers"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.3</span> <span>Commercial web crawlers</span> </div> </a> <ul id="toc-Commercial_web_crawlers-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Open-source_crawlers" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Open-source_crawlers"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.4</span> <span>Open-source crawlers</span> </div> </a> <ul id="toc-Open-source_crawlers-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> <span class="vector-toc-numb">10</span> <span>See also</span> </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">11</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Further_reading" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Further_reading"> <div class="vector-toc-text"> <span class="vector-toc-numb">12</span> <span>Further reading</span> </div> </a> <ul id="toc-Further_reading-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Web crawler</span></h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 47 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-47" aria-hidden="true" ><span class="vector-icon mw-ui-icon-language-progressive mw-ui-icon-wikimedia-language-progressive"></span> <span class="vector-dropdown-label-text">47 languages</span> </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-af mw-list-item"><a href="https://af.wikipedia.org/wiki/Webkruiper" title="Webkruiper – Afrikaans" lang="af" hreflang="af" data-title="Webkruiper" data-language-autonym="Afrikaans" data-language-local-name="Afrikaans" class="interlanguage-link-target"><span>Afrikaans</span></a></li><li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D8%B2%D8%A7%D8%AD%D9%81_%D9%88%D9%8A%D8%A8" title="زاحف ويب – Arabic" lang="ar" hreflang="ar" data-title="زاحف ويب" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target"><span>العربية</span></a></li><li class="interlanguage-link interwiki-az mw-list-item"><a href="https://az.wikipedia.org/wiki/Axtar%C4%B1%C5%9F_botu" title="Axtarış botu – Azerbaijani" lang="az" hreflang="az" data-title="Axtarış botu" data-language-autonym="Azərbaycanca" data-language-local-name="Azerbaijani" class="interlanguage-link-target"><span>Azərbaycanca</span></a></li><li class="interlanguage-link interwiki-bar mw-list-item"><a href="https://bar.wikipedia.org/wiki/Webcrawler" title="Webcrawler – Bavarian" lang="bar" hreflang="bar" data-title="Webcrawler" data-language-autonym="Boarisch" data-language-local-name="Bavarian" class="interlanguage-link-target"><span>Boarisch</span></a></li><li class="interlanguage-link interwiki-ca mw-list-item"><a href="https://ca.wikipedia.org/wiki/Aranya_web" title="Aranya web – Catalan" lang="ca" hreflang="ca" data-title="Aranya web" data-language-autonym="Català" data-language-local-name="Catalan" class="interlanguage-link-target"><span>Català</span></a></li><li class="interlanguage-link interwiki-cs mw-list-item"><a href="https://cs.wikipedia.org/wiki/Web_crawler" title="Web crawler – Czech" lang="cs" hreflang="cs" data-title="Web crawler" data-language-autonym="Čeština" data-language-local-name="Czech" class="interlanguage-link-target"><span>Čeština</span></a></li><li class="interlanguage-link interwiki-cy mw-list-item"><a href="https://cy.wikipedia.org/wiki/Ymgripiwr_gwe" title="Ymgripiwr gwe – Welsh" lang="cy" hreflang="cy" data-title="Ymgripiwr gwe" data-language-autonym="Cymraeg" data-language-local-name="Welsh" class="interlanguage-link-target"><span>Cymraeg</span></a></li><li class="interlanguage-link interwiki-ary mw-list-item"><a href="https://ary.wikipedia.org/wiki/%D8%B1%D8%AA%D9%8A%D9%84%D8%A9_(%D8%A8%D9%88%D8%AA)" title="رتيلة (بوت) – Moroccan Arabic" lang="ary" hreflang="ary" data-title="رتيلة (بوت)" data-language-autonym="الدارجة" data-language-local-name="Moroccan Arabic" class="interlanguage-link-target"><span>الدارجة</span></a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Webcrawler" title="Webcrawler – German" lang="de" hreflang="de" data-title="Webcrawler" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target"><span>Deutsch</span></a></li><li class="interlanguage-link interwiki-et mw-list-item"><a href="https://et.wikipedia.org/wiki/Veebi%C3%A4mblik" title="Veebiämblik – Estonian" lang="et" hreflang="et" data-title="Veebiämblik" data-language-autonym="Eesti" data-language-local-name="Estonian" class="interlanguage-link-target"><span>Eesti</span></a></li><li class="interlanguage-link interwiki-el mw-list-item"><a href="https://el.wikipedia.org/wiki/%CE%91%CE%BD%CE%B9%CF%87%CE%BD%CE%B5%CF%85%CF%84%CE%AE%CF%82_%CE%B9%CF%83%CF%84%CE%BF%CF%8D" title="Ανιχνευτής ιστού – Greek" lang="el" hreflang="el" data-title="Ανιχνευτής ιστού" data-language-autonym="Ελληνικά" data-language-local-name="Greek" class="interlanguage-link-target"><span>Ελληνικά</span></a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Ara%C3%B1a_web" title="Araña web – Spanish" lang="es" hreflang="es" data-title="Araña web" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target"><span>Español</span></a></li><li class="interlanguage-link interwiki-eu mw-list-item"><a href="https://eu.wikipedia.org/wiki/Web_crawler" title="Web crawler – Basque" lang="eu" hreflang="eu" data-title="Web crawler" data-language-autonym="Euskara" data-language-local-name="Basque" class="interlanguage-link-target"><span>Euskara</span></a></li><li class="interlanguage-link interwiki-fa mw-list-item"><a href="https://fa.wikipedia.org/wiki/%D8%AE%D8%B2%D9%86%D8%AF%D9%87_%D9%88%D8%A8" title="خزنده وب – Persian" lang="fa" hreflang="fa" data-title="خزنده وب" data-language-autonym="فارسی" data-language-local-name="Persian" class="interlanguage-link-target"><span>فارسی</span></a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/Robot_d%27indexation" title="Robot d'indexation – French" lang="fr" hreflang="fr" data-title="Robot d'indexation" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target"><span>Français</span></a></li><li class="interlanguage-link interwiki-ko mw-list-item"><a href="https://ko.wikipedia.org/wiki/%EC%9B%B9_%ED%81%AC%EB%A1%A4%EB%9F%AC" title="웹 크롤러 – Korean" lang="ko" hreflang="ko" data-title="웹 크롤러" data-language-autonym="한국어" data-language-local-name="Korean" class="interlanguage-link-target"><span>한국어</span></a></li><li class="interlanguage-link interwiki-hy mw-list-item"><a href="https://hy.wikipedia.org/wiki/%D5%88%D6%80%D5%B8%D5%B6%D5%B8%D5%B2%D5%A1%D5%AF%D5%A1%D5%B6_%D5%BC%D5%B8%D5%A2%D5%B8%D5%BF" title="Որոնողական ռոբոտ – Armenian" lang="hy" hreflang="hy" data-title="Որոնողական ռոբոտ" data-language-autonym="Հայերեն" data-language-local-name="Armenian" class="interlanguage-link-target"><span>Հայերեն</span></a></li><li class="interlanguage-link interwiki-hr mw-list-item"><a href="https://hr.wikipedia.org/wiki/Web_pauk" title="Web pauk – Croatian" lang="hr" hreflang="hr" data-title="Web pauk" data-language-autonym="Hrvatski" data-language-local-name="Croatian" class="interlanguage-link-target"><span>Hrvatski</span></a></li><li class="interlanguage-link interwiki-id mw-list-item"><a href="https://id.wikipedia.org/wiki/Perayap_web" title="Perayap web – Indonesian" lang="id" hreflang="id" data-title="Perayap web" data-language-autonym="Bahasa Indonesia" data-language-local-name="Indonesian" class="interlanguage-link-target"><span>Bahasa Indonesia</span></a></li><li class="interlanguage-link interwiki-ia mw-list-item"><a href="https://ia.wikipedia.org/wiki/Rastreator_de_rete" title="Rastreator de rete – Interlingua" lang="ia" hreflang="ia" data-title="Rastreator de rete" data-language-autonym="Interlingua" data-language-local-name="Interlingua" class="interlanguage-link-target"><span>Interlingua</span></a></li><li class="interlanguage-link interwiki-it mw-list-item"><a href="https://it.wikipedia.org/wiki/Crawler" title="Crawler – Italian" lang="it" hreflang="it" data-title="Crawler" data-language-autonym="Italiano" data-language-local-name="Italian" class="interlanguage-link-target"><span>Italiano</span></a></li><li class="interlanguage-link interwiki-he mw-list-item"><a href="https://he.wikipedia.org/wiki/%D7%96%D7%97%D7%9C%D7%9F_%D7%A8%D7%A9%D7%AA" title="זחלן רשת – Hebrew" lang="he" hreflang="he" data-title="זחלן רשת" data-language-autonym="עברית" data-language-local-name="Hebrew" class="interlanguage-link-target"><span>עברית</span></a></li><li class="interlanguage-link interwiki-lv mw-list-item"><a href="https://lv.wikipedia.org/wiki/T%C4%ABmek%C4%BCa_r%C4%81pu%C4%BCprogramma" title="Tīmekļa rāpuļprogramma – Latvian" lang="lv" hreflang="lv" data-title="Tīmekļa rāpuļprogramma" data-language-autonym="Latviešu" data-language-local-name="Latvian" class="interlanguage-link-target"><span>Latviešu</span></a></li><li class="interlanguage-link interwiki-lt mw-list-item"><a href="https://lt.wikipedia.org/wiki/Interneto_robotas" title="Interneto robotas – Lithuanian" lang="lt" hreflang="lt" data-title="Interneto robotas" data-language-autonym="Lietuvių" data-language-local-name="Lithuanian" class="interlanguage-link-target"><span>Lietuvių</span></a></li><li class="interlanguage-link interwiki-hu mw-list-item"><a href="https://hu.wikipedia.org/wiki/Keres%C5%91robot" title="Keresőrobot – Hungarian" lang="hu" hreflang="hu" data-title="Keresőrobot" data-language-autonym="Magyar" data-language-local-name="Hungarian" class="interlanguage-link-target"><span>Magyar</span></a></li><li class="interlanguage-link interwiki-ms mw-list-item"><a href="https://ms.wikipedia.org/wiki/Perangkak_web" title="Perangkak web – Malay" lang="ms" hreflang="ms" data-title="Perangkak web" data-language-autonym="Bahasa Melayu" data-language-local-name="Malay" class="interlanguage-link-target"><span>Bahasa Melayu</span></a></li><li class="interlanguage-link interwiki-nl mw-list-item"><a href="https://nl.wikipedia.org/wiki/Spider" title="Spider – Dutch" lang="nl" hreflang="nl" data-title="Spider" data-language-autonym="Nederlands" data-language-local-name="Dutch" class="interlanguage-link-target"><span>Nederlands</span></a></li><li class="interlanguage-link interwiki-nds-nl mw-list-item"><a href="https://nds-nl.wikipedia.org/wiki/Webkroeper" title="Webkroeper – Low Saxon" lang="nds-NL" hreflang="nds-NL" data-title="Webkroeper" data-language-autonym="Nedersaksies" data-language-local-name="Low Saxon" class="interlanguage-link-target"><span>Nedersaksies</span></a></li><li class="interlanguage-link interwiki-ja mw-list-item"><a href="https://ja.wikipedia.org/wiki/%E3%82%AF%E3%83%AD%E3%83%BC%E3%83%A9" title="クローラ – Japanese" lang="ja" hreflang="ja" data-title="クローラ" data-language-autonym="日本語" data-language-local-name="Japanese" class="interlanguage-link-target"><span>日本語</span></a></li><li class="interlanguage-link interwiki-no mw-list-item"><a href="https://no.wikipedia.org/wiki/S%C3%B8kerobot" title="Søkerobot – Norwegian Bokmål" lang="nb" hreflang="nb" data-title="Søkerobot" data-language-autonym="Norsk bokmål" data-language-local-name="Norwegian Bokmål" class="interlanguage-link-target"><span>Norsk bokmål</span></a></li><li class="interlanguage-link interwiki-nn mw-list-item"><a href="https://nn.wikipedia.org/wiki/S%C3%B8kerobot" title="Søkerobot – Norwegian Nynorsk" lang="nn" hreflang="nn" data-title="Søkerobot" data-language-autonym="Norsk nynorsk" data-language-local-name="Norwegian Nynorsk" class="interlanguage-link-target"><span>Norsk nynorsk</span></a></li><li class="interlanguage-link interwiki-mhr mw-list-item"><a href="https://mhr.wikipedia.org/wiki/%D0%9A%D1%8B%D1%87%D0%B0%D0%BB%D1%88%D0%B5_%D1%80%D0%BE%D0%B1%D0%BE%D1%82" title="Кычалше робот – Eastern Mari" lang="mhr" hreflang="mhr" data-title="Кычалше робот" data-language-autonym="Олык марий" data-language-local-name="Eastern Mari" class="interlanguage-link-target"><span>Олык марий</span></a></li><li class="interlanguage-link interwiki-pl mw-list-item"><a href="https://pl.wikipedia.org/wiki/Robot_internetowy" title="Robot internetowy – Polish" lang="pl" hreflang="pl" data-title="Robot internetowy" data-language-autonym="Polski" data-language-local-name="Polish" class="interlanguage-link-target"><span>Polski</span></a></li><li class="interlanguage-link interwiki-pt mw-list-item"><a href="https://pt.wikipedia.org/wiki/Rastreador_web" title="Rastreador web – Portuguese" lang="pt" hreflang="pt" data-title="Rastreador web" data-language-autonym="Português" data-language-local-name="Portuguese" class="interlanguage-link-target"><span>Português</span></a></li><li class="interlanguage-link interwiki-ro mw-list-item"><a href="https://ro.wikipedia.org/wiki/Robot_de_c%C4%83utare" title="Robot de căutare – Romanian" lang="ro" hreflang="ro" data-title="Robot de căutare" data-language-autonym="Română" data-language-local-name="Romanian" class="interlanguage-link-target"><span>Română</span></a></li><li class="interlanguage-link interwiki-qu mw-list-item"><a href="https://qu.wikipedia.org/wiki/Web_aysakuq" title="Web aysakuq – Quechua" lang="qu" hreflang="qu" data-title="Web aysakuq" data-language-autonym="Runa Simi" data-language-local-name="Quechua" class="interlanguage-link-target"><span>Runa Simi</span></a></li><li class="interlanguage-link interwiki-ru mw-list-item"><a href="https://ru.wikipedia.org/wiki/%D0%9F%D0%BE%D0%B8%D1%81%D0%BA%D0%BE%D0%B2%D1%8B%D0%B9_%D1%80%D0%BE%D0%B1%D0%BE%D1%82" title="Поисковый робот – Russian" lang="ru" hreflang="ru" data-title="Поисковый робот" data-language-autonym="Русский" data-language-local-name="Russian" class="interlanguage-link-target"><span>Русский</span></a></li><li class="interlanguage-link interwiki-simple mw-list-item"><a href="https://simple.wikipedia.org/wiki/Web_crawler" title="Web crawler – Simple English" lang="en-simple" hreflang="en-simple" data-title="Web crawler" data-language-autonym="Simple English" data-language-local-name="Simple English" class="interlanguage-link-target"><span>Simple English</span></a></li><li class="interlanguage-link interwiki-sr mw-list-item"><a href="https://sr.wikipedia.org/wiki/Veb-popisiva%C4%8D" title="Veb-popisivač – Serbian" lang="sr" hreflang="sr" data-title="Veb-popisivač" data-language-autonym="Српски / srpski" data-language-local-name="Serbian" class="interlanguage-link-target"><span>Српски / srpski</span></a></li><li class="interlanguage-link interwiki-fi mw-list-item"><a href="https://fi.wikipedia.org/wiki/Hakurobotti" title="Hakurobotti – Finnish" lang="fi" hreflang="fi" data-title="Hakurobotti" data-language-autonym="Suomi" data-language-local-name="Finnish" class="interlanguage-link-target"><span>Suomi</span></a></li><li class="interlanguage-link interwiki-sv mw-list-item"><a href="https://sv.wikipedia.org/wiki/Spindel_(internet)" title="Spindel (internet) – Swedish" lang="sv" hreflang="sv" data-title="Spindel (internet)" data-language-autonym="Svenska" data-language-local-name="Swedish" class="interlanguage-link-target"><span>Svenska</span></a></li><li class="interlanguage-link interwiki-ta mw-list-item"><a href="https://ta.wikipedia.org/wiki/%E0%AE%B5%E0%AE%B2%E0%AF%88_%E0%AE%8A%E0%AE%B0%E0%AF%8D%E0%AE%A4%E0%AE%BF" title="வலை ஊர்தி – Tamil" lang="ta" hreflang="ta" data-title="வலை ஊர்தி" data-language-autonym="தமிழ்" data-language-local-name="Tamil" class="interlanguage-link-target"><span>தமிழ்</span></a></li><li class="interlanguage-link interwiki-th mw-list-item"><a href="https://th.wikipedia.org/wiki/%E0%B9%80%E0%B8%A7%E0%B9%87%E0%B8%9A%E0%B8%84%E0%B8%A3%E0%B8%AD%E0%B8%A7%E0%B9%8C%E0%B9%80%E0%B8%A5%E0%B8%AD%E0%B8%A3%E0%B9%8C" title="เว็บครอว์เลอร์ – Thai" lang="th" hreflang="th" data-title="เว็บครอว์เลอร์" data-language-autonym="ไทย" data-language-local-name="Thai" class="interlanguage-link-target"><span>ไทย</span></a></li><li class="interlanguage-link interwiki-tr mw-list-item"><a href="https://tr.wikipedia.org/wiki/Arama_robotu" title="Arama robotu – Turkish" lang="tr" hreflang="tr" data-title="Arama robotu" data-language-autonym="Türkçe" data-language-local-name="Turkish" class="interlanguage-link-target"><span>Türkçe</span></a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%9F%D0%BE%D1%88%D1%83%D0%BA%D0%BE%D0%B2%D0%B8%D0%B9_%D1%80%D0%BE%D0%B1%D0%BE%D1%82" title="Пошуковий робот – Ukrainian" lang="uk" hreflang="uk" data-title="Пошуковий робот" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target"><span>Українська</span></a></li><li class="interlanguage-link interwiki-zh-classical mw-list-item"><a href="https://zh-classical.wikipedia.org/wiki/%E7%88%AC%E7%B6%B2%E8%9F%B2%E6%A2%B0" title="爬網蟲械 – Literary Chinese" lang="lzh" hreflang="lzh" data-title="爬網蟲械" data-language-autonym="文言" data-language-local-name="Literary Chinese" class="interlanguage-link-target"><span>文言</span></a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E7%B6%B2%E8%B7%AF%E7%88%AC%E8%9F%B2" title="網路爬蟲 – Chinese" lang="zh" hreflang="zh" data-title="網路爬蟲" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target"><span>中文</span></a></li> </ul> <div class="after-portlet after-portlet-lang"><span class="wb-langlinks-edit wb-langlinks-link"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q45842#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></span></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Web_crawler" title="View the content page [c]" accesskey="c"><span>Article</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Web_crawler" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t"><span>Talk</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Web_crawler"><span>Read</span></a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Web_crawler&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Web_crawler&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Web_crawler"><span>Read</span></a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Web_crawler&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Web_crawler&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Web_crawler" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Web_crawler" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u"><span>Upload file</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Web_crawler&oldid=1257967634" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Web_crawler&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Web_crawler&id=1257967634&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FWeb_crawler"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FWeb_crawler"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Web_crawler&action=show-download-screen" title="Download this page as a PDF file"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Web_crawler&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q45842" title="Structured data on this page hosted by Wikidata [g]" accesskey="g"><span>Wikidata item</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Software which systematically browses the World Wide Web</div> <style data-mw-deduplicate="TemplateStyles:r1236090951">.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}</style><div role="note" class="hatnote navigation-not-searchable"><span>This article is about the internet bot. For the search engine, see <a href="/wiki/WebCrawler" title="WebCrawler">WebCrawler</a>.</span> <span>"Web spider" redirects here. Not to be confused with <a href="/wiki/Spider_web" title="Spider web">Spider web</a>.</span> <span>"Spiderbot" redirects here. For the video game, see <a href="/wiki/Arac_(video_game)" title="Arac (video game)">Arac (video game)</a>.</span></div> <p class="mw-empty-elt"> </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:WebCrawlerArchitecture.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/220px-WebCrawlerArchitecture.svg.png" decoding="async" width="220" height="168" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/330px-WebCrawlerArchitecture.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/440px-WebCrawlerArchitecture.svg.png 2x" data-file-width="500" data-file-height="382" /></a><figcaption>Architecture of a Web crawler</figcaption></figure> <p>A <b>Web crawler</b>, sometimes called a <b>spider</b> or <b>spiderbot</b> and often shortened to <b>crawler</b>, is an <a href="/wiki/Internet_bot" title="Internet bot">Internet bot</a> that systematically browses the <a href="/wiki/World_Wide_Web" title="World Wide Web">World Wide Web</a> and that is typically operated by search engines for the purpose of <a href="/wiki/Web_indexing" title="Web indexing">Web indexing</a> (<i>web spidering</i>).<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> </p><p>Web <a href="/wiki/Search_engine" title="Search engine">search engines</a> and some other <a href="/wiki/Website" title="Website">websites</a> use Web crawling or spidering <a href="/wiki/Software" title="Software">software</a> to update their <a href="/wiki/Web_content" title="Web content">web content</a> or indices of other sites' web content. Web crawlers copy pages for processing by a search engine, which <a href="/wiki/Index_(search_engine)" class="mw-redirect" title="Index (search engine)">indexes</a> the downloaded pages so that users can search more efficiently. </p><p>Crawlers consume resources on visited systems and often visit sites unprompted. Issues of schedule, load, and "politeness" come into play when large collections of pages are accessed. Mechanisms exist for public sites not wishing to be crawled to make this known to the crawling agent. For example, including a <code><a href="/wiki/Robots.txt" title="Robots.txt">robots.txt</a></code> file can request <a href="/wiki/Software_agent" title="Software agent">bots</a> to index only parts of a website, or nothing at all. </p><p>The number of Internet pages is extremely large; even the largest crawlers fall short of making a complete index. For this reason, search engines struggled to give relevant search results in the early years of the World Wide Web, before 2000. Today, relevant results are given almost instantly. </p><p>Crawlers can validate <a href="/wiki/Hyperlink" title="Hyperlink">hyperlinks</a> and <a href="/wiki/HTML" title="HTML">HTML</a> code. They can also be used for <a href="/wiki/Web_scraping" title="Web scraping">web scraping</a> and <a href="/wiki/Data-driven_programming" title="Data-driven programming">data-driven programming</a>. </p> <meta property="mw:PageProp/toc" /> <div class="mw-heading mw-heading2"><h2 id="Nomenclature">Nomenclature</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=1" title="Edit section: Nomenclature"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A web crawler is also known as a <i>spider</i>,<sup id="cite_ref-spekta_2-0" class="reference"><a href="#cite_note-spekta-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> an <i>ant</i>, an <i>automatic indexer</i>,<sup id="cite_ref-3" class="reference"><a href="#cite_note-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> or (in the <a href="/wiki/FOAF_(software)" class="mw-redirect" title="FOAF (software)">FOAF</a> software context) a <i>Web scutter</i>.<sup id="cite_ref-4" class="reference"><a href="#cite_note-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Overview">Overview</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=2" title="Edit section: Overview"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A Web crawler starts with a list of <a href="/wiki/Uniform_Resource_Locator" class="mw-redirect" title="Uniform Resource Locator">URLs</a> to visit. Those first URLs are called the <i>seeds</i>. As the crawler visits these URLs, by communicating with <a href="/wiki/Web_server" title="Web server">web servers</a> that respond to those URLs, it identifies all the <a href="/wiki/Hyperlink" title="Hyperlink">hyperlinks</a> in the retrieved web pages and adds them to the list of URLs to visit, called the <i><a href="/wiki/Crawl_frontier" title="Crawl frontier">crawl frontier</a></i>. URLs from the frontier are <a href="/wiki/Recursion" title="Recursion">recursively</a> visited according to a set of policies. If the crawler is performing archiving of <a href="/wiki/Website" title="Website">websites</a> (or <a href="/wiki/Web_archiving" title="Web archiving">web archiving</a>), it copies and saves the information as it goes. The archives are usually stored in such a way they can be viewed, read and navigated as if they were on the live web, but are preserved as 'snapshots'.<sup id="cite_ref-GoogleBooks-9237380_5-0" class="reference"><a href="#cite_note-GoogleBooks-9237380-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> </p><p>The archive is known as the <i>repository</i> and is designed to store and manage the collection of <a href="/wiki/Web_page" title="Web page">web pages</a>. The <a href="/wiki/Repository_(version_control)" title="Repository (version control)">repository</a> only stores <a href="/wiki/HTML" title="HTML">HTML</a> pages and these pages are stored as distinct files. A repository is similar to any other system that stores data, like a modern-day database. The only difference is that a repository does not need all the functionality offered by a database system. The repository stores the most recent version of the web page retrieved by the crawler.<sup class="noprint Inline-Template Template-Fact" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed"><span title="This claim needs references to reliable sources. (February 2023)">citation needed</span></a></i>]</sup> </p><p>The large volume implies the crawler can only download a limited number of the Web pages within a given time, so it needs to prioritize its downloads. The high rate of change can imply the pages might have already been updated or even deleted. </p><p>The number of possible URLs crawled being generated by server-side software has also made it difficult for web crawlers to avoid retrieving <a href="/wiki/Duplicate_content" title="Duplicate content">duplicate content</a>. Endless combinations of <a href="/wiki/HTTP" title="HTTP">HTTP</a> GET (URL-based) parameters exist, of which only a small selection will actually return unique content. For example, a simple online photo gallery may offer three options to users, as specified through HTTP GET parameters in the URL. If there exist four ways to sort images, three choices of <a href="/wiki/Thumbnail" title="Thumbnail">thumbnail</a> size, two file formats, and an option to disable user-provided content, then the same set of content can be accessed with 48 different URLs, all of which may be linked on the site. This <a href="/wiki/Mathematical_combination" class="mw-redirect" title="Mathematical combination">mathematical combination</a> creates a problem for crawlers, as they must sort through endless combinations of relatively minor scripted changes in order to retrieve unique content. </p><p>As Edwards <i>et al.</i> noted, "Given that the <a href="/wiki/Bandwidth_(computing)" title="Bandwidth (computing)">bandwidth</a> for conducting crawls is neither infinite nor free, it is becoming essential to crawl the Web in not only a scalable, but efficient way, if some reasonable measure of quality or freshness is to be maintained."<sup id="cite_ref-edwards2001_6-0" class="reference"><a href="#cite_note-edwards2001-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> A crawler must carefully choose at each step which pages to visit next. </p> <div class="mw-heading mw-heading2"><h2 id="Crawling_policy">Crawling policy</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=3" title="Edit section: Crawling policy"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The behavior of a Web crawler is the outcome of a combination of policies:<sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup> </p> <ul><li>a <i>selection policy</i> which states the pages to download,</li> <li>a <i>re-visit policy</i> which states when to check for changes to the pages,</li> <li>a <i>politeness policy</i> that states how to avoid overloading <a href="/wiki/Websites" class="mw-redirect" title="Websites">websites</a>.</li> <li>a <i>parallelization policy</i> that states how to coordinate distributed web crawlers.</li></ul> <div class="mw-heading mw-heading3"><h3 id="Selection_policy">Selection policy</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=4" title="Edit section: Selection policy"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Given the current size of the Web, even large search engines cover only a portion of the publicly available part. A 2009 study showed even large-scale <a href="/wiki/Search_engines" class="mw-redirect" title="Search engines">search engines</a> index no more than 40–70% of the indexable Web;<sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> a previous study by <a href="/wiki/Steve_Lawrence_(computer_scientist)" title="Steve Lawrence (computer scientist)">Steve Lawrence</a> and <a href="/wiki/Lee_Giles" title="Lee Giles">Lee Giles</a> showed that no <a href="/wiki/Search_engine_indexing" title="Search engine indexing">search engine indexed</a> more than 16% of the Web in 1999.<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> As a crawler always downloads just a fraction of the <a href="/wiki/Web_pages" class="mw-redirect" title="Web pages">Web pages</a>, it is highly desirable for the downloaded fraction to contain the most relevant pages and not just a random sample of the Web. </p><p>This requires a metric of importance for prioritizing Web pages. The importance of a page is a function of its <a href="/wiki/Intrinsic_and_extrinsic_properties_(philosophy)" title="Intrinsic and extrinsic properties (philosophy)">intrinsic</a> quality, its popularity in terms of links or visits, and even of its URL (the latter is the case of <a href="/wiki/Vertical_search" title="Vertical search">vertical search engines</a> restricted to a single <a href="/wiki/Top-level_domain" title="Top-level domain">top-level domain</a>, or search engines restricted to a fixed Web site). Designing a good selection policy has an added difficulty: it must work with partial information, as the complete set of Web pages is not known during crawling. </p><p>Junghoo Cho <i>et al.</i> made the first study on policies for crawling scheduling. Their data set was a 180,000-pages crawl from the <kbd>stanford.edu</kbd> domain, in which a crawling simulation was done with different strategies.<sup id="cite_ref-10" class="reference"><a href="#cite_note-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> The ordering metrics tested were <a href="/wiki/Breadth-first_search" title="Breadth-first search">breadth-first</a>, <a href="/wiki/Backlink" title="Backlink">backlink</a> count and partial <a href="/wiki/PageRank" title="PageRank">PageRank</a> calculations. One of the conclusions was that if the crawler wants to download pages with high Pagerank early during the crawling process, then the partial Pagerank strategy is the better, followed by breadth-first and backlink-count. However, these results are for just a single domain. Cho also wrote his PhD dissertation at Stanford on web crawling.<sup id="cite_ref-11" class="reference"><a href="#cite_note-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> </p><p>Najork and Wiener performed an actual crawl on 328 million pages, using breadth-first ordering.<sup id="cite_ref-12" class="reference"><a href="#cite_note-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> They found that a breadth-first crawl captures pages with high Pagerank early in the crawl (but they did not compare this strategy against other strategies). The explanation given by the authors for this result is that "the most important pages have many links to them from numerous hosts, and those links will be found early, regardless of on which host or page the crawl originates." </p><p>Abiteboul designed a crawling strategy based on an <a href="/wiki/Algorithm" title="Algorithm">algorithm</a> called OPIC (On-line Page Importance Computation).<sup id="cite_ref-13" class="reference"><a href="#cite_note-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> In OPIC, each page is given an initial sum of "cash" that is distributed equally among the pages it points to. It is similar to a PageRank computation, but it is faster and is only done in one step. An OPIC-driven crawler downloads first the pages in the crawling frontier with higher amounts of "cash". Experiments were carried in a 100,000-pages synthetic graph with a power-law distribution of in-links. However, there was no comparison with other strategies nor experiments in the real Web. </p><p>Boldi <i>et al.</i> used simulation on subsets of the Web of 40 million pages from the <kbd>.it</kbd> domain and 100 million pages from the WebBase crawl, testing breadth-first against depth-first, random ordering and an omniscient strategy. The comparison was based on how well PageRank computed on a partial crawl approximates the true PageRank value. Some visits that accumulate PageRank very quickly (most notably, breadth-first and the omniscient visit) provide very poor progressive approximations.<sup id="cite_ref-14" class="reference"><a href="#cite_note-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-15" class="reference"><a href="#cite_note-15"><span class="cite-bracket">[</span>15<span class="cite-bracket">]</span></a></sup> </p><p>Baeza-Yates <i>et al.</i> used simulation on two subsets of the Web of 3 million pages from the <kbd>.gr</kbd> and <kbd>.cl</kbd> domain, testing several crawling strategies.<sup id="cite_ref-baeza2005_16-0" class="reference"><a href="#cite_note-baeza2005-16"><span class="cite-bracket">[</span>16<span class="cite-bracket">]</span></a></sup> They showed that both the OPIC strategy and a strategy that uses the length of the per-site queues are better than <a href="/wiki/Breadth-first_search" title="Breadth-first search">breadth-first</a> crawling, and that it is also very effective to use a previous crawl, when it is available, to guide the current one. </p><p>Daneshpajouh <i>et al.</i> designed a community based algorithm for discovering good seeds.<sup id="cite_ref-17" class="reference"><a href="#cite_note-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup> Their method crawls web pages with high PageRank from different communities in less iteration in comparison with crawl starting from random seeds. One can extract good seed from a previously-crawled-Web graph using this new method. Using these seeds, a new crawl can be very effective. </p> <div class="mw-heading mw-heading4"><h4 id="Restricting_followed_links">Restricting followed links</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=5" title="Edit section: Restricting followed links"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A crawler may only want to seek out HTML pages and avoid all other <a href="/wiki/Internet_media_type" class="mw-redirect" title="Internet media type">MIME types</a>. In order to request only HTML resources, a crawler may make an HTTP HEAD request to determine a Web resource's MIME type before requesting the entire resource with a GET request. To avoid making numerous HEAD requests, a crawler may examine the URL and only request a resource if the URL ends with certain characters such as .html, .htm, .asp, .aspx, .php, .jsp, .jspx or a slash. This strategy may cause numerous HTML Web resources to be unintentionally skipped. </p><p>Some crawlers may also avoid requesting any resources that have a <a href="/wiki/Query_string" title="Query string">"?"</a> in them (are dynamically produced) in order to avoid <a href="/wiki/Spider_trap" title="Spider trap">spider traps</a> that may cause the crawler to download an infinite number of URLs from a Web site. This strategy is unreliable if the site uses <a href="/wiki/URL_rewriting" class="mw-redirect" title="URL rewriting">URL rewriting</a> to simplify its URLs. </p> <div class="mw-heading mw-heading4"><h4 id="URL_normalization">URL normalization</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=6" title="Edit section: URL normalization"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/URL_normalization" class="mw-redirect" title="URL normalization">URL normalization</a></div> <p>Crawlers usually perform some type of <a href="/wiki/URL_normalization" class="mw-redirect" title="URL normalization">URL normalization</a> in order to avoid crawling the same resource more than once. The term <i>URL normalization</i>, also called <i>URL canonicalization</i>, refers to the process of modifying and standardizing a URL in a consistent manner. There are several types of normalization that may be performed including conversion of URLs to lowercase, removal of "." and ".." segments, and adding trailing slashes to the non-empty path component.<sup id="cite_ref-18" class="reference"><a href="#cite_note-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading4"><h4 id="Path-ascending_crawling">Path-ascending crawling</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=7" title="Edit section: Path-ascending crawling"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Some crawlers intend to download/upload as many resources as possible from a particular web site. So <i>path-ascending crawler</i> was introduced that would ascend to every path in each URL that it intends to crawl.<sup id="cite_ref-19" class="reference"><a href="#cite_note-19"><span class="cite-bracket">[</span>19<span class="cite-bracket">]</span></a></sup> For example, when given a seed URL of http://llama.org/hamster/monkey/page.html, it will attempt to crawl /hamster/monkey/, /hamster/, and /. Cothey found that a path-ascending crawler was very effective in finding isolated resources, or resources for which no inbound link would have been found in regular crawling. </p> <div class="mw-heading mw-heading4"><h4 id="Focused_crawling">Focused crawling</h4><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=8" title="Edit section: Focused crawling"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Focused_crawler" title="Focused crawler">Focused crawler</a></div> <p>The importance of a page for a crawler can also be expressed as a function of the similarity of a page to a given query. Web crawlers that attempt to download pages that are similar to each other are called <b>focused crawler</b> or <b>topical crawlers</b>. The concepts of topical and focused crawling were first introduced by <a href="/wiki/Filippo_Menczer" title="Filippo Menczer">Filippo Menczer</a><sup id="cite_ref-20" class="reference"><a href="#cite_note-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-21" class="reference"><a href="#cite_note-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup> and by Soumen Chakrabarti <i>et al.</i><sup id="cite_ref-22" class="reference"><a href="#cite_note-22"><span class="cite-bracket">[</span>22<span class="cite-bracket">]</span></a></sup> </p><p>The main problem in focused crawling is that in the context of a Web crawler, we would like to be able to predict the similarity of the text of a given page to the query before actually downloading the page. A possible predictor is the anchor text of links; this was the approach taken by Pinkerton<sup id="cite_ref-pinkerton1994_23-0" class="reference"><a href="#cite_note-pinkerton1994-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup> in the first web crawler of the early days of the Web. Diligenti <i>et al.</i><sup id="cite_ref-24" class="reference"><a href="#cite_note-24"><span class="cite-bracket">[</span>24<span class="cite-bracket">]</span></a></sup> propose using the complete content of the pages already visited to infer the similarity between the driving query and the pages that have not been visited yet. The performance of a focused crawling depends mostly on the richness of links in the specific topic being searched, and a focused crawling usually relies on a general Web search engine for providing starting points. </p> <div class="mw-heading mw-heading5"><h5 id="Academic_focused_crawler">Academic focused crawler</h5><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=9" title="Edit section: Academic focused crawler"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>An example of the <a href="/wiki/Focused_crawlers" class="mw-redirect" title="Focused crawlers">focused crawlers</a> are academic crawlers, which crawls free-access academic related documents, such as the <i>citeseerxbot</i>, which is the crawler of <a href="/wiki/CiteSeer" class="mw-redirect" title="CiteSeer">CiteSeer</a><sup>X</sup> search engine. Other academic search engines are <a href="/wiki/Google_Scholar" title="Google Scholar">Google Scholar</a> and <a href="/wiki/Microsoft_Academic_Search" title="Microsoft Academic Search">Microsoft Academic Search</a> etc. Because most academic papers are published in <a href="/wiki/PDF" title="PDF">PDF</a> formats, such kind of crawler is particularly interested in crawling PDF, <a href="/wiki/PostScript" title="PostScript">PostScript</a> files, <a href="/wiki/Microsoft_Word" title="Microsoft Word">Microsoft Word</a> including their <a href="/wiki/Zipped_file" class="mw-redirect" title="Zipped file">zipped</a> formats. Because of this, general open-source crawlers, such as <a href="/wiki/Heritrix" title="Heritrix">Heritrix</a>, must be customized to filter out other <a href="/wiki/MIME_types" class="mw-redirect" title="MIME types">MIME types</a>, or a <a href="/wiki/Middleware" title="Middleware">middleware</a> is used to extract these documents out and import them to the focused crawl database and repository.<sup id="cite_ref-25" class="reference"><a href="#cite_note-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup> Identifying whether these documents are academic or not is challenging and can add a significant overhead to the crawling process, so this is performed as a post crawling process using <a href="/wiki/Machine_learning" title="Machine learning">machine learning</a> or <a href="/wiki/Regular_expression" title="Regular expression">regular expression</a> algorithms. These academic documents are usually obtained from home pages of faculties and students or from publication page of research institutes. Because academic documents make up only a small fraction of all web pages, a good seed selection is important in boosting the efficiencies of these web crawlers.<sup id="cite_ref-26" class="reference"><a href="#cite_note-26"><span class="cite-bracket">[</span>26<span class="cite-bracket">]</span></a></sup> Other academic crawlers may download plain text and <a href="/wiki/HTML" title="HTML">HTML</a> files, that contains <a href="/wiki/Metadata" title="Metadata">metadata</a> of academic papers, such as titles, papers, and abstracts. This increases the overall number of papers, but a significant fraction may not provide free PDF downloads. </p> <div class="mw-heading mw-heading5"><h5 id="Semantic_focused_crawler">Semantic focused crawler</h5><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=10" title="Edit section: Semantic focused crawler"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Another type of focused crawlers is semantic focused crawler, which makes use of <a href="/wiki/Domain_ontology" class="mw-redirect" title="Domain ontology">domain ontologies</a> to represent topical maps and link Web pages with relevant ontological concepts for the selection and categorization purposes.<sup id="cite_ref-27" class="reference"><a href="#cite_note-27"><span class="cite-bracket">[</span>27<span class="cite-bracket">]</span></a></sup> In addition, ontologies can be automatically updated in the crawling process. Dong et al.<sup id="cite_ref-28" class="reference"><a href="#cite_note-28"><span class="cite-bracket">[</span>28<span class="cite-bracket">]</span></a></sup> introduced such an ontology-learning-based crawler using a <a href="/wiki/Support-vector_machine" class="mw-redirect" title="Support-vector machine">support-vector machine</a> to update the content of ontological concepts when crawling Web pages. </p> <div class="mw-heading mw-heading3"><h3 id="Re-visit_policy">Re-visit policy</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=11" title="Edit section: Re-visit policy"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The Web has a very dynamic nature, and crawling a fraction of the Web can take weeks or months. By the time a Web crawler has finished its crawl, many events could have happened, including creations, updates, and deletions. </p><p>From the search engine's point of view, there is a cost associated with not detecting an event, and thus having an outdated copy of a resource. The most-used cost functions are freshness and age.<sup id="cite_ref-29" class="reference"><a href="#cite_note-29"><span class="cite-bracket">[</span>29<span class="cite-bracket">]</span></a></sup> </p><p><b>Freshness</b>: This is a binary measure that indicates whether the local copy is accurate or not. The freshness of a page <i>p</i> in the repository at time <i>t</i> is defined as: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{p}(t)={\begin{cases}1&{\rm {if}}~p~{\rm {~is~equal~to~the~local~copy~at~time}}~t\\0&{\rm {otherwise}}\end{cases}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>F</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>p</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>t</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>{</mo> <mtable columnalign="left left" rowspacing=".2em" columnspacing="1em" displaystyle="false"> <mtr> <mtd> <mn>1</mn> </mtd> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">f</mi> </mrow> </mrow> <mtext> </mtext> <mi>p</mi> <mtext> </mtext> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mtext> </mtext> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">s</mi> <mtext> </mtext> <mi mathvariant="normal">e</mi> <mi mathvariant="normal">q</mi> <mi mathvariant="normal">u</mi> <mi mathvariant="normal">a</mi> <mi mathvariant="normal">l</mi> <mtext> </mtext> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">o</mi> <mtext> </mtext> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">h</mi> <mi mathvariant="normal">e</mi> <mtext> </mtext> <mi mathvariant="normal">l</mi> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">c</mi> <mi mathvariant="normal">a</mi> <mi mathvariant="normal">l</mi> <mtext> </mtext> <mi mathvariant="normal">c</mi> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">p</mi> <mi mathvariant="normal">y</mi> <mtext> </mtext> <mi mathvariant="normal">a</mi> <mi mathvariant="normal">t</mi> <mtext> </mtext> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">m</mi> <mi mathvariant="normal">e</mi> </mrow> </mrow> <mtext> </mtext> <mi>t</mi> </mtd> </mtr> <mtr> <mtd> <mn>0</mn> </mtd> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">h</mi> <mi mathvariant="normal">e</mi> <mi mathvariant="normal">r</mi> <mi mathvariant="normal">w</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">s</mi> <mi mathvariant="normal">e</mi> </mrow> </mrow> </mtd> </mtr> </mtable> <mo fence="true" stretchy="true" symmetric="true"></mo> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle F_{p}(t)={\begin{cases}1&{\rm {if}}~p~{\rm {~is~equal~to~the~local~copy~at~time}}~t\\0&{\rm {otherwise}}\end{cases}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/542c7504faa720e2407afc0ec6e80639896ba14d" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:52.379ex; height:6.176ex;" alt="{\displaystyle F_{p}(t)={\begin{cases}1&{\rm {if}}~p~{\rm {~is~equal~to~the~local~copy~at~time}}~t\\0&{\rm {otherwise}}\end{cases}}}"></span></dd></dl> <p><b>Age</b>: This is a measure that indicates how outdated the local copy is. The age of a page <i>p</i> in the repository, at time <i>t</i> is defined as: </p> <dl><dd><span class="mwe-math-element"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle A_{p}(t)={\begin{cases}0&{\rm {if}}~p~{\rm {~is~not~modified~at~time}}~t\\t-{\rm {modification~time~of}}~p&{\rm {otherwise}}\end{cases}}}"> <semantics> <mrow class="MJX-TeXAtom-ORD"> <mstyle displaystyle="true" scriptlevel="0"> <msub> <mi>A</mi> <mrow class="MJX-TeXAtom-ORD"> <mi>p</mi> </mrow> </msub> <mo stretchy="false">(</mo> <mi>t</mi> <mo stretchy="false">)</mo> <mo>=</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow> <mo>{</mo> <mtable columnalign="left left" rowspacing=".2em" columnspacing="1em" displaystyle="false"> <mtr> <mtd> <mn>0</mn> </mtd> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">f</mi> </mrow> </mrow> <mtext> </mtext> <mi>p</mi> <mtext> </mtext> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mtext> </mtext> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">s</mi> <mtext> </mtext> <mi mathvariant="normal">n</mi> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">t</mi> <mtext> </mtext> <mi mathvariant="normal">m</mi> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">d</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">f</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">e</mi> <mi mathvariant="normal">d</mi> <mtext> </mtext> <mi mathvariant="normal">a</mi> <mi mathvariant="normal">t</mi> <mtext> </mtext> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">m</mi> <mi mathvariant="normal">e</mi> </mrow> </mrow> <mtext> </mtext> <mi>t</mi> </mtd> </mtr> <mtr> <mtd> <mi>t</mi> <mo>−</mo> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">m</mi> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">d</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">f</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">c</mi> <mi mathvariant="normal">a</mi> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">n</mi> <mtext> </mtext> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">m</mi> <mi mathvariant="normal">e</mi> <mtext> </mtext> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">f</mi> </mrow> </mrow> <mtext> </mtext> <mi>p</mi> </mtd> <mtd> <mrow class="MJX-TeXAtom-ORD"> <mrow class="MJX-TeXAtom-ORD"> <mi mathvariant="normal">o</mi> <mi mathvariant="normal">t</mi> <mi mathvariant="normal">h</mi> <mi mathvariant="normal">e</mi> <mi mathvariant="normal">r</mi> <mi mathvariant="normal">w</mi> <mi mathvariant="normal">i</mi> <mi mathvariant="normal">s</mi> <mi mathvariant="normal">e</mi> </mrow> </mrow> </mtd> </mtr> </mtable> <mo fence="true" stretchy="true" symmetric="true"></mo> </mrow> </mrow> </mstyle> </mrow> <annotation encoding="application/x-tex">{\displaystyle A_{p}(t)={\begin{cases}0&{\rm {if}}~p~{\rm {~is~not~modified~at~time}}~t\\t-{\rm {modification~time~of}}~p&{\rm {otherwise}}\end{cases}}}</annotation> </semantics> </math></span><img src="https://wikimedia.org/api/rest_v1/media/math/render/svg/d1dfda1dc1dd66ec242ff2e4a515588d6611532a" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:67.807ex; height:6.176ex;" alt="{\displaystyle A_{p}(t)={\begin{cases}0&{\rm {if}}~p~{\rm {~is~not~modified~at~time}}~t\\t-{\rm {modification~time~of}}~p&{\rm {otherwise}}\end{cases}}}"></span></dd></dl> <p><a href="/wiki/Edward_G._Coffman,_Jr." class="mw-redirect" title="Edward G. Coffman, Jr.">Coffman</a> <i>et al.</i> worked with a definition of the objective of a Web crawler that is equivalent to freshness, but use a different wording: they propose that a crawler must minimize the fraction of time pages remain outdated. They also noted that the problem of Web crawling can be modeled as a multiple-queue, single-server polling system, on which the Web crawler is the server and the Web sites are the queues. Page modifications are the arrival of the customers, and switch-over times are the interval between page accesses to a single Web site. Under this model, mean waiting time for a customer in the polling system is equivalent to the average age for the Web crawler.<sup id="cite_ref-coffman_30-0" class="reference"><a href="#cite_note-coffman-30"><span class="cite-bracket">[</span>30<span class="cite-bracket">]</span></a></sup> </p><p>The objective of the crawler is to keep the average freshness of pages in its collection as high as possible, or to keep the average age of pages as low as possible. These objectives are not equivalent: in the first case, the crawler is just concerned with how many pages are outdated, while in the second case, the crawler is concerned with how old the local copies of pages are. </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Web_Crawling_Freshness_Age.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/8/86/Web_Crawling_Freshness_Age.png/220px-Web_Crawling_Freshness_Age.png" decoding="async" width="220" height="155" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/86/Web_Crawling_Freshness_Age.png/330px-Web_Crawling_Freshness_Age.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/86/Web_Crawling_Freshness_Age.png/440px-Web_Crawling_Freshness_Age.png 2x" data-file-width="500" data-file-height="353" /></a><figcaption>Evolution of Freshness and Age in a web crawler</figcaption></figure> <p>Two simple re-visiting policies were studied by Cho and Garcia-Molina:<sup id="cite_ref-cho2003_31-0" class="reference"><a href="#cite_note-cho2003-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup> </p> <ul><li>Uniform policy: This involves re-visiting all pages in the collection with the same frequency, regardless of their rates of change.</li> <li>Proportional policy: This involves re-visiting more often the pages that change more frequently. The visiting frequency is directly proportional to the (estimated) change frequency.</li></ul> <p>In both cases, the repeated crawling order of pages can be done either in a random or a fixed order. </p><p>Cho and Garcia-Molina proved the surprising result that, in terms of average freshness, the uniform policy outperforms the proportional policy in both a simulated Web and a real Web crawl. Intuitively, the reasoning is that, as web crawlers have a limit to how many pages they can crawl in a given time frame, (1) they will allocate too many new crawls to rapidly changing pages at the expense of less frequently updating pages, and (2) the freshness of rapidly changing pages lasts for shorter period than that of less frequently changing pages. In other words, a proportional policy allocates more resources to crawling frequently updating pages, but experiences less overall freshness time from them. </p><p>To improve freshness, the crawler should penalize the elements that change too often.<sup id="cite_ref-cho2003a_32-0" class="reference"><a href="#cite_note-cho2003a-32"><span class="cite-bracket">[</span>32<span class="cite-bracket">]</span></a></sup> The optimal re-visiting policy is neither the uniform policy nor the proportional policy. The optimal method for keeping average freshness high includes ignoring the pages that change too often, and the optimal for keeping average age low is to use access frequencies that monotonically (and sub-linearly) increase with the rate of change of each page. In both cases, the optimal is closer to the uniform policy than to the proportional policy: as <a href="/wiki/Edward_G._Coffman,_Jr." class="mw-redirect" title="Edward G. Coffman, Jr.">Coffman</a> <i>et al.</i> note, "in order to minimize the expected obsolescence time, the accesses to any particular page should be kept as evenly spaced as possible".<sup id="cite_ref-coffman_30-1" class="reference"><a href="#cite_note-coffman-30"><span class="cite-bracket">[</span>30<span class="cite-bracket">]</span></a></sup> Explicit formulas for the re-visit policy are not attainable in general, but they are obtained numerically, as they depend on the distribution of page changes. Cho and Garcia-Molina show that the exponential distribution is a good fit for describing page changes,<sup id="cite_ref-cho2003a_32-1" class="reference"><a href="#cite_note-cho2003a-32"><span class="cite-bracket">[</span>32<span class="cite-bracket">]</span></a></sup> while <a href="/wiki/Panos_Ipeirotis" title="Panos Ipeirotis">Ipeirotis</a> <i>et al.</i> show how to use statistical tools to discover parameters that affect this distribution.<sup id="cite_ref-33" class="reference"><a href="#cite_note-33"><span class="cite-bracket">[</span>33<span class="cite-bracket">]</span></a></sup> The re-visiting policies considered here regard all pages as homogeneous in terms of quality ("all pages on the Web are worth the same"), something that is not a realistic scenario, so further information about the Web page quality should be included to achieve a better crawling policy. </p> <div class="mw-heading mw-heading3"><h3 id="Politeness_policy">Politeness policy</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=12" title="Edit section: Politeness policy"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Crawlers can retrieve data much quicker and in greater depth than human searchers, so they can have a crippling impact on the performance of a site. If a single crawler is performing multiple requests per second and/or downloading large files, a server can have a hard time keeping up with requests from multiple crawlers. </p><p>As noted by Koster, the use of Web crawlers is useful for a number of tasks, but comes with a price for the general community.<sup id="cite_ref-34" class="reference"><a href="#cite_note-34"><span class="cite-bracket">[</span>34<span class="cite-bracket">]</span></a></sup> The costs of using Web crawlers include: </p> <ul><li>network resources, as crawlers require considerable bandwidth and operate with a high degree of parallelism during a long period of time;</li> <li>server overload, especially if the frequency of accesses to a given server is too high;</li> <li>poorly written crawlers, which can crash servers or routers, or which download pages they cannot handle; and</li> <li>personal crawlers that, if deployed by too many users, can disrupt networks and Web servers.</li></ul> <p>A partial solution to these problems is the <a href="/wiki/Robots_Exclusion_Standard" class="mw-redirect" title="Robots Exclusion Standard">robots exclusion protocol</a>, also known as the robots.txt protocol that is a standard for administrators to indicate which parts of their Web servers should not be accessed by crawlers.<sup id="cite_ref-35" class="reference"><a href="#cite_note-35"><span class="cite-bracket">[</span>35<span class="cite-bracket">]</span></a></sup> This standard does not include a suggestion for the interval of visits to the same server, even though this interval is the most effective way of avoiding server overload. Recently commercial search engines like <a href="/wiki/Google.com" class="mw-redirect" title="Google.com">Google</a>, <a href="/wiki/Ask.com" title="Ask.com">Ask Jeeves</a>, <a href="/wiki/Bing_(search_engine)" class="mw-redirect" title="Bing (search engine)">MSN</a> and <a href="/wiki/Yahoo!_Search" class="mw-redirect" title="Yahoo! Search">Yahoo! Search</a> are able to use an extra "Crawl-delay:" parameter in the <a href="/wiki/Robots.txt" title="Robots.txt">robots.txt</a> file to indicate the number of seconds to delay between requests. </p><p>The first proposed interval between successive pageloads was 60 seconds.<sup id="cite_ref-36" class="reference"><a href="#cite_note-36"><span class="cite-bracket">[</span>36<span class="cite-bracket">]</span></a></sup> However, if pages were downloaded at this rate from a website with more than 100,000 pages over a perfect connection with zero latency and infinite bandwidth, it would take more than 2 months to download only that entire Web site; also, only a fraction of the resources from that Web server would be used. </p><p>Cho uses 10 seconds as an interval for accesses,<sup id="cite_ref-cho2003_31-1" class="reference"><a href="#cite_note-cho2003-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup> and the WIRE crawler uses 15 seconds as the default.<sup id="cite_ref-baeza2002_37-0" class="reference"><a href="#cite_note-baeza2002-37"><span class="cite-bracket">[</span>37<span class="cite-bracket">]</span></a></sup> The MercatorWeb crawler follows an adaptive politeness policy: if it took <i>t</i> seconds to download a document from a given server, the crawler waits for 10<i>t</i> seconds before downloading the next page.<sup id="cite_ref-38" class="reference"><a href="#cite_note-38"><span class="cite-bracket">[</span>38<span class="cite-bracket">]</span></a></sup> Dill <i>et al.</i> use 1 second.<sup id="cite_ref-39" class="reference"><a href="#cite_note-39"><span class="cite-bracket">[</span>39<span class="cite-bracket">]</span></a></sup> </p><p>For those using Web crawlers for research purposes, a more detailed cost-benefit analysis is needed and ethical considerations should be taken into account when deciding where to crawl and how fast to crawl.<sup id="cite_ref-40" class="reference"><a href="#cite_note-40"><span class="cite-bracket">[</span>40<span class="cite-bracket">]</span></a></sup> </p><p>Anecdotal evidence from access logs shows that access intervals from known crawlers vary between 20 seconds and 3–4 minutes. It is worth noticing that even when being very polite, and taking all the safeguards to avoid overloading Web servers, some complaints from Web server administrators are received. <a href="/wiki/Sergey_Brin" title="Sergey Brin">Sergey Brin</a> and <a href="/wiki/Larry_Page" title="Larry Page">Larry Page</a> noted in 1998, "... running a crawler which connects to more than half a million servers ... generates a fair amount of e-mail and phone calls. Because of the vast number of people coming on line, there are always those who do not know what a crawler is, because this is the first one they have seen."<sup id="cite_ref-brin1998_41-0" class="reference"><a href="#cite_note-brin1998-41"><span class="cite-bracket">[</span>41<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Parallelization_policy">Parallelization policy</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=13" title="Edit section: Parallelization policy"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Distributed_web_crawling" title="Distributed web crawling">Distributed web crawling</a></div> <p>A <a href="/wiki/Parallel_computing" title="Parallel computing">parallel</a> crawler is a crawler that runs multiple processes in parallel. The goal is to maximize the download rate while minimizing the overhead from parallelization and to avoid repeated downloads of the same page. To avoid downloading the same page more than once, the crawling system requires a policy for assigning the new URLs discovered during the crawling process, as the same URL can be found by two different crawling processes. </p> <div class="mw-heading mw-heading2"><h2 id="Architectures">Architectures</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=14" title="Edit section: Architectures"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <figure typeof="mw:File/Thumb"><a href="/wiki/File:WebCrawlerArchitecture.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/300px-WebCrawlerArchitecture.svg.png" decoding="async" width="300" height="229" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/450px-WebCrawlerArchitecture.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/600px-WebCrawlerArchitecture.svg.png 2x" data-file-width="500" data-file-height="382" /></a><figcaption>High-level architecture of a standard Web crawler</figcaption></figure> <p>A crawler must not only have a good crawling strategy, as noted in the previous sections, but it should also have a highly optimized architecture. </p><p>Shkapenyuk and Suel noted that:<sup id="cite_ref-shkapenyuk2002_42-0" class="reference"><a href="#cite_note-shkapenyuk2002-42"><span class="cite-bracket">[</span>42<span class="cite-bracket">]</span></a></sup> </p> <style data-mw-deduplicate="TemplateStyles:r1244412712">.mw-parser-output .templatequote{overflow:hidden;margin:1em 0;padding:0 32px}.mw-parser-output .templatequotecite{line-height:1.5em;text-align:left;margin-top:0}@media(min-width:500px){.mw-parser-output .templatequotecite{padding-left:1.6em}}</style><blockquote class="templatequote"><p>While it is fairly easy to build a slow crawler that downloads a few pages per second for a short period of time, building a high-performance system that can download hundreds of millions of pages over several weeks presents a number of challenges in system design, I/O and network efficiency, and robustness and manageability.</p></blockquote> <p>Web crawlers are a central part of search engines, and details on their algorithms and architecture are kept as business secrets. When crawler designs are published, there is often an important lack of detail that prevents others from reproducing the work. There are also emerging concerns about "<a href="/wiki/Spamdexing" title="Spamdexing">search engine spamming</a>", which prevent major search engines from publishing their ranking algorithms. </p> <div class="mw-heading mw-heading2"><h2 id="Security">Security</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=15" title="Edit section: Security"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>While most of the website owners are keen to have their pages indexed as broadly as possible to have strong presence in <a href="/wiki/Web_search_engine" class="mw-redirect" title="Web search engine">search engines</a>, web crawling can also have <a href="/wiki/Unintended_consequences" title="Unintended consequences">unintended consequences</a> and lead to a <a href="/wiki/Web_application_security" class="mw-redirect" title="Web application security">compromise</a> or <a href="/wiki/Data_breach" title="Data breach">data breach</a> if a search engine indexes resources that should not be publicly available, or pages revealing potentially vulnerable versions of software. </p> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Main article: <a href="/wiki/Google_hacking" title="Google hacking">Google hacking</a></div> <p>Apart from standard <a href="/wiki/Web_application_security" class="mw-redirect" title="Web application security">web application security</a> recommendations website owners can reduce their exposure to opportunistic hacking by only allowing search engines to index the public parts of their websites (with <a href="/wiki/Robots_exclusion_standard" class="mw-redirect" title="Robots exclusion standard">robots.txt</a>) and explicitly blocking them from indexing transactional parts (login pages, private pages, etc.). </p> <div class="mw-heading mw-heading2"><h2 id="Crawler_identification">Crawler identification</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=16" title="Edit section: Crawler identification"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Web crawlers typically identify themselves to a Web server by using the <a href="/wiki/User_agent" title="User agent">User-agent</a> field of an <a href="/wiki/HTTP" title="HTTP">HTTP</a> request. Web site administrators typically examine their <a href="/wiki/Web_server" title="Web server">Web servers</a>' log and use the user agent field to determine which crawlers have visited the web server and how often. The user agent field may include a <a href="/wiki/Uniform_Resource_Locator" class="mw-redirect" title="Uniform Resource Locator">URL</a> where the Web site administrator may find out more information about the crawler. Examining Web server log is tedious task, and therefore some administrators use tools to identify, track and verify Web crawlers. <a href="/wiki/Spambots" class="mw-redirect" title="Spambots">Spambots</a> and other malicious Web crawlers are unlikely to place identifying information in the user agent field, or they may mask their identity as a browser or other well-known crawler. </p><p>Web site administrators prefer Web crawlers to identify themselves so that they can contact the owner if needed. In some cases, crawlers may be accidentally trapped in a <a href="/wiki/Spider_trap" title="Spider trap">crawler trap</a> or they may be overloading a Web server with requests, and the owner needs to stop the crawler. Identification is also useful for administrators that are interested in knowing when they may expect their Web pages to be indexed by a particular <a href="/wiki/Web_search_engine" class="mw-redirect" title="Web search engine">search engine</a>. </p> <div class="mw-heading mw-heading2"><h2 id="Crawling_the_deep_web">Crawling the deep web</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=17" title="Edit section: Crawling the deep web"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>A vast amount of web pages lie in the <a href="/wiki/Deep_Web_(search_indexing)" class="mw-redirect" title="Deep Web (search indexing)">deep or invisible web</a>.<sup id="cite_ref-43" class="reference"><a href="#cite_note-43"><span class="cite-bracket">[</span>43<span class="cite-bracket">]</span></a></sup> These pages are typically only accessible by submitting queries to a database, and regular crawlers are unable to find these pages if there are no links that point to them. Google's <a href="/wiki/Sitemaps" title="Sitemaps">Sitemaps</a> protocol and <a href="/wiki/Mod_oai" title="Mod oai">mod oai</a><sup id="cite_ref-44" class="reference"><a href="#cite_note-44"><span class="cite-bracket">[</span>44<span class="cite-bracket">]</span></a></sup> are intended to allow discovery of these <a href="/wiki/Deep_web" title="Deep web">deep-Web</a> resources. </p><p>Deep web crawling also multiplies the number of web links to be crawled. Some crawlers only take some of the URLs in <code><a href="URL"></code> form. In some cases, such as the <a href="/wiki/Googlebot" title="Googlebot">Googlebot</a>, Web crawling is done on all text contained inside the hypertext content, tags, or text. </p><p>Strategic approaches may be taken to target deep Web content. With a technique called <a href="/wiki/Screen_scraping" class="mw-redirect" title="Screen scraping">screen scraping</a>, specialized software may be customized to automatically and repeatedly query a given Web form with the intention of aggregating the resulting data. Such software can be used to span multiple Web forms across multiple Websites. Data extracted from the results of one Web form submission can be taken and applied as input to another Web form thus establishing continuity across the Deep Web in a way not possible with traditional web crawlers.<sup id="cite_ref-45" class="reference"><a href="#cite_note-45"><span class="cite-bracket">[</span>45<span class="cite-bracket">]</span></a></sup> </p><p>Pages built on <a href="/wiki/AJAX" class="mw-redirect" title="AJAX">AJAX</a> are among those causing problems to web crawlers. <a href="/wiki/Google_search_engine" class="mw-redirect" title="Google search engine">Google</a> has proposed a format of AJAX calls that their bot can recognize and index.<sup id="cite_ref-46" class="reference"><a href="#cite_note-46"><span class="cite-bracket">[</span>46<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Visual_vs_programmatic_crawlers">Visual vs programmatic crawlers</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=18" title="Edit section: Visual vs programmatic crawlers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>There are a number of "visual web scraper/crawler" products available on the web which will crawl pages and structure data into columns and rows based on the users requirements. One of the main difference between a classic and a visual crawler is the level of programming ability required to set up a crawler. The latest generation of "visual scrapers" remove the majority of the programming skill needed to be able to program and start a crawl to scrape web data. </p><p>The visual scraping/crawling method relies on the user "teaching" a piece of crawler technology, which then follows patterns in semi-structured data sources. The dominant method for teaching a visual crawler is by highlighting data in a browser and training columns and rows. While the technology is not new, for example it was the basis of Needlebase which has been bought by Google (as part of a larger acquisition of ITA Labs<sup id="cite_ref-47" class="reference"><a href="#cite_note-47"><span class="cite-bracket">[</span>47<span class="cite-bracket">]</span></a></sup>), there is continued growth and investment in this area by investors and end-users.<sup class="noprint Inline-Template Template-Fact" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed"><span title="This claim needs references to reliable sources. (June 2021)">citation needed</span></a></i>]</sup> </p> <div class="mw-heading mw-heading2"><h2 id="List_of_web_crawlers">List of web crawlers</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=19" title="Edit section: List of web crawlers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236090951"><div role="note" class="hatnote navigation-not-searchable">Further information: <a href="/wiki/List_of_search_engine_software" title="List of search engine software">List of search engine software</a></div> <p>The following is a list of published crawler architectures for general-purpose crawlers (excluding focused web crawlers), with a brief description that includes the names given to the different components and outstanding features: </p> <div class="mw-heading mw-heading3"><h3 id="Historical_web_crawlers">Historical web crawlers</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=20" title="Edit section: Historical web crawlers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20020106130042/http://www.wolfbot.com/">WolfBot</a> was a massively multi threaded crawler built in 2001 by Mani Singh a Civil Engineering graduate from the University of California at Davis.</li> <li><a href="/wiki/World_Wide_Web_Worm" title="World Wide Web Worm">World Wide Web Worm</a> was a crawler used to build a simple index of document titles and URLs. The index could be searched by using the <kbd><a href="/wiki/Grep" title="Grep">grep</a></kbd> <a href="/wiki/Unix" title="Unix">Unix</a> command.</li> <li>Yahoo! Slurp was the name of the <a href="/wiki/Yahoo!" class="mw-redirect" title="Yahoo!">Yahoo!</a> Search crawler until Yahoo! contracted with <a href="/wiki/Microsoft" title="Microsoft">Microsoft</a> to use <a href="/wiki/Bingbot" title="Bingbot">Bingbot</a> instead.</li></ul> <div class="mw-heading mw-heading3"><h3 id="In-house_web_crawlers">In-house web crawlers</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=21" title="Edit section: In-house web crawlers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li>Applebot is <a href="/wiki/Apple_(company)" class="mw-redirect" title="Apple (company)">Apple</a>'s web crawler. It supports <a href="/wiki/Siri" title="Siri">Siri</a> and other products.<sup id="cite_ref-48" class="reference"><a href="#cite_note-48"><span class="cite-bracket">[</span>48<span class="cite-bracket">]</span></a></sup></li> <li><a href="/wiki/Bingbot" title="Bingbot">Bingbot</a> is the name of Microsoft's <a href="/wiki/Bing_(search_engine)" class="mw-redirect" title="Bing (search engine)">Bing</a> webcrawler. It replaced <i><a href="/wiki/Msnbot" title="Msnbot">Msnbot</a></i>.</li> <li>Baiduspider is <a href="/wiki/Baidu" title="Baidu">Baidu</a>'s web crawler.</li> <li>DuckDuckBot is <a href="/wiki/DuckDuckGo" title="DuckDuckGo">DuckDuckGo</a>'s web crawler.</li> <li><a href="/wiki/Googlebot" title="Googlebot">Googlebot</a> is described in some detail, but the reference is only about an early version of its architecture, which was written in C++ and <a href="/wiki/Python_(programming_language)" title="Python (programming language)">Python</a>. The crawler was integrated with the indexing process, because text parsing was done for full-text indexing and also for URL extraction. There is a URL server that sends lists of URLs to be fetched by several crawling processes. During parsing, the URLs found were passed to a URL server that checked if the URL have been previously seen. If not, the URL was added to the queue of the URL server.</li> <li><a href="/wiki/WebCrawler" title="WebCrawler">WebCrawler</a> was used to build the first publicly available full-text index of a subset of the Web. It was based on <a href="/wiki/Libwww" title="Libwww">lib-WWW</a> to download pages, and another program to parse and order URLs for breadth-first exploration of the Web graph. It also included a real-time crawler that followed links based on the similarity of the anchor text with the provided query.</li> <li><a href="/wiki/WebFountain" class="mw-redirect" title="WebFountain">WebFountain</a> is a distributed, modular crawler similar to Mercator but written in C++.</li> <li><a href="/wiki/Xenon_(program)" title="Xenon (program)">Xenon</a> is a web crawler used by government tax authorities to detect fraud.<sup id="cite_ref-Norton-2007-01-25_49-0" class="reference"><a href="#cite_note-Norton-2007-01-25-49"><span class="cite-bracket">[</span>49<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-Canada-2017-04-11_50-0" class="reference"><a href="#cite_note-Canada-2017-04-11-50"><span class="cite-bracket">[</span>50<span class="cite-bracket">]</span></a></sup></li></ul> <div class="mw-heading mw-heading3"><h3 id="Commercial_web_crawlers">Commercial web crawlers</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=22" title="Edit section: Commercial web crawlers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The following web crawlers are available, for a price:: </p> <ul><li><a href="/wiki/Diffbot" title="Diffbot">Diffbot</a> - programmatic general web crawler, available as an <a href="/wiki/API" title="API">API</a></li> <li><a href="/wiki/SortSite" title="SortSite">SortSite</a> - crawler for analyzing websites, available for <a href="/wiki/Microsoft_Windows" title="Microsoft Windows">Windows</a> and <a href="/wiki/Macintosh_operating_systems" class="mw-redirect" title="Macintosh operating systems">Mac OS</a></li> <li>Swiftbot - <a href="/wiki/Swiftype" title="Swiftype">Swiftype</a>'s web crawler, available as <a href="/wiki/Software_as_a_service" title="Software as a service">software as a service</a></li></ul> <div class="mw-heading mw-heading3"><h3 id="Open-source_crawlers">Open-source crawlers</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=23" title="Edit section: Open-source crawlers"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Apache_Nutch" title="Apache Nutch">Apache Nutch</a> is a highly extensible and scalable web crawler written in Java and released under an <a href="/wiki/Apache_License" title="Apache License">Apache License</a>. It is based on <a href="/wiki/Apache_Hadoop" title="Apache Hadoop">Apache Hadoop</a> and can be used with <a href="/wiki/Apache_Solr" title="Apache Solr">Apache Solr</a> or <a href="/wiki/Elasticsearch" title="Elasticsearch">Elasticsearch</a>.</li> <li><a href="/wiki/Grub_(search_engine)" title="Grub (search engine)">Grub</a> was an open source distributed search crawler that <a href="/wiki/Wikia_Search" title="Wikia Search">Wikia Search</a> used to crawl the web.</li> <li><a href="/wiki/Heritrix" title="Heritrix">Heritrix</a> is the <a href="/wiki/Internet_Archive" title="Internet Archive">Internet Archive</a>'s archival-quality crawler, designed for archiving periodic snapshots of a large portion of the Web. It was written in <a href="/wiki/Java_(programming_language)" title="Java (programming language)">Java</a>.</li> <li><a href="/wiki/Ht-//dig" class="mw-redirect" title="Ht-//dig">ht://Dig</a> includes a Web crawler in its indexing engine.</li> <li><a href="/wiki/HTTrack" title="HTTrack">HTTrack</a> uses a Web crawler to create a mirror of a web site for off-line viewing. It is written in <a href="/wiki/C_(programming_language)" title="C (programming language)">C</a> and released under the GPL.</li> <li>Norconex Web Crawler is a highly extensible Web Crawler written in <a href="/wiki/Java_(programming_language)" title="Java (programming language)">Java</a> and released under an <a href="/wiki/Apache_License" title="Apache License">Apache License</a>. It can be used with many repositories such as <a href="/wiki/Apache_Solr" title="Apache Solr">Apache Solr</a>, <a href="/wiki/Elasticsearch" title="Elasticsearch">Elasticsearch</a>, <a href="/wiki/Azure_Cognitive_Search" title="Azure Cognitive Search">Microsoft Azure Cognitive Search</a>, <a href="/wiki/Amazon_CloudSearch" class="mw-redirect" title="Amazon CloudSearch">Amazon CloudSearch</a> and more.</li> <li><a href="/wiki/MnoGoSearch" title="MnoGoSearch">mnoGoSearch</a> is a crawler, indexer and a search engine written in C and licensed under the GPL (*NIX machines only)</li> <li><a href="/wiki/Open_Search_Server" class="mw-redirect" title="Open Search Server">Open Search Server</a> is a search engine and web crawler software release under the GPL.</li> <li><a href="/wiki/Scrapy" title="Scrapy">Scrapy</a>, an open source webcrawler framework, written in python (licensed under <a href="/wiki/BSD_License" class="mw-redirect" title="BSD License">BSD</a>).</li> <li><a href="/wiki/Seeks" title="Seeks">Seeks</a>, a free distributed search engine (licensed under <a href="/wiki/GNU_Affero_General_Public_License" title="GNU Affero General Public License">AGPL</a>).</li> <li><a href="/wiki/StormCrawler" title="StormCrawler">StormCrawler</a>, a collection of resources for building low-latency, scalable web crawlers on <a href="/wiki/Storm_(event_processor)" class="mw-redirect" title="Storm (event processor)">Apache Storm</a> (Apache License).</li> <li><a href="/wiki/TkWWW_Robot" class="mw-redirect" title="TkWWW Robot">tkWWW Robot</a>, a crawler based on the <a href="/wiki/TkWWW" title="TkWWW">tkWWW</a> web browser (licensed under GPL).</li> <li><a href="/wiki/Wget" title="Wget">GNU Wget</a> is a <a href="/wiki/Command_line_interface" class="mw-redirect" title="Command line interface">command-line</a>-operated crawler written in <a href="/wiki/C_(programming_language)" title="C (programming language)">C</a> and released under the <a href="/wiki/GNU_General_Public_License" title="GNU General Public License">GPL</a>. It is typically used to mirror Web and FTP sites.</li> <li><a href="/wiki/Xapian" title="Xapian">Xapian</a>, a search crawler engine, written in c++.</li> <li><a href="/wiki/YaCy" title="YaCy">YaCy</a>, a free distributed search engine, built on principles of peer-to-peer networks (licensed under GPL).</li></ul> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=24" title="Edit section: See also"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li><a href="/wiki/Automatic_indexing" title="Automatic indexing">Automatic indexing</a></li> <li><a href="/wiki/Gnutella_crawler" class="mw-redirect" title="Gnutella crawler">Gnutella crawler</a></li> <li><a href="/wiki/Web_archiving" title="Web archiving">Web archiving</a></li> <li><a href="/wiki/Webgraph" title="Webgraph">Webgraph</a></li> <li><a href="/wiki/Website_mirroring_software" class="mw-redirect" title="Website mirroring software">Website mirroring software</a></li> <li><a href="/wiki/Search_Engine_Scraping" class="mw-redirect" title="Search Engine Scraping">Search Engine Scraping</a></li> <li><a href="/wiki/Web_scraping" title="Web scraping">Web scraping</a></li></ul> <div class="mw-heading mw-heading2"><h2 id="References">References</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=25" title="Edit section: References"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em ;"> <ol class="references"> <li id="cite_note-1"><span class="mw-cite-backlink"><b><a href="#cite_ref-1">^</a></b></span> <span class="reference-text"><style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20211206205907/https://webbrowsersintroduction.com/">"Web Crawlers: Browsing the Web"</a>. Archived from <a rel="nofollow" class="external text" href="https://webbrowsersintroduction.com/">the original</a> on 6 December 2021.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Web+Crawlers%3A+Browsing+the+Web&rft_id=https%3A%2F%2Fwebbrowsersintroduction.com%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-spekta-2"><span class="mw-cite-backlink"><b><a href="#cite_ref-spekta_2-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFSpetka" class="citation web cs1">Spetka, Scott. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20040903174942/http://archive.ncsa.uiuc.edu/SDG/IT94/Proceedings/Agents/spetka/spetka.html">"The TkWWW Robot: Beyond Browsing"</a>. <a href="/wiki/National_Center_for_Supercomputing_Applications" title="National Center for Supercomputing Applications">NCSA</a>. Archived from <a rel="nofollow" class="external text" href="http://archive.ncsa.uiuc.edu/SDG/IT94/Proceedings/Agents/spetka/spetka.html">the original</a> on 3 September 2004<span class="reference-accessdate">. Retrieved <span class="nowrap">21 November</span> 2010</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=The+TkWWW+Robot%3A+Beyond+Browsing&rft.pub=NCSA&rft.aulast=Spetka&rft.aufirst=Scott&rft_id=http%3A%2F%2Farchive.ncsa.uiuc.edu%2FSDG%2FIT94%2FProceedings%2FAgents%2Fspetka%2Fspetka.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-3"><span class="mw-cite-backlink"><b><a href="#cite_ref-3">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKobayashi,_M.Takeda,_K.2000" class="citation journal cs1">Kobayashi, M. & Takeda, K. (2000). "Information retrieval on the web". <i>ACM Computing Surveys</i>. <b>32</b> (2): 144–173. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.126.6094">10.1.1.126.6094</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F358923.358934">10.1145/358923.358934</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:3710903">3710903</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=ACM+Computing+Surveys&rft.atitle=Information+retrieval+on+the+web&rft.volume=32&rft.issue=2&rft.pages=144-173&rft.date=2000&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.126.6094%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A3710903%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F358923.358934&rft.au=Kobayashi%2C+M.&rft.au=Takeda%2C+K.&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-4">^</a></b></span> <span class="reference-text">See <a rel="nofollow" class="external text" href="http://wiki.foaf-project.org/w/Scutter">definition of scutter on FOAF Project's wiki</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20091213213920/http://wiki.foaf-project.org/w/Scutter">Archived</a> 13 December 2009 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a></span> </li> <li id="cite_note-GoogleBooks-9237380-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-GoogleBooks-9237380_5-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMasanès2007" class="citation book cs1">Masanès, Julien (15 February 2007). <a rel="nofollow" class="external text" href="https://www.springer.com/gp/book/9783540233381"><i>Web Archiving</i></a>. Springer. p. 1. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-54046332-0" title="Special:BookSources/978-3-54046332-0"><bdi>978-3-54046332-0</bdi></a><span class="reference-accessdate">. Retrieved <span class="nowrap">24 April</span> 2014</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=book&rft.btitle=Web+Archiving&rft.pages=1&rft.pub=Springer&rft.date=2007-02-15&rft.isbn=978-3-54046332-0&rft.aulast=Masan%C3%A8s&rft.aufirst=Julien&rft_id=https%3A%2F%2Fwww.springer.com%2Fgp%2Fbook%2F9783540233381&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-edwards2001-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-edwards2001_6-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFEdwards,_J.;_McCurley,_K._S.;_and_Tomlin,_J._A.2001" class="citation book cs1">Edwards, J.; McCurley, K. S.; and Tomlin, J. A. (2001). "An adaptive model for optimizing performance of an incremental web crawler". <a rel="nofollow" class="external text" href="https://web.archive.org/web/20140625233510/http://www10.org/cdrom/papers/210/index.html"><i>Proceedings of the 10th international conference on World Wide Web</i></a>. pp. 106–113. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.1018.1506">10.1.1.1018.1506</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F371920.371960">10.1145/371920.371960</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-1581133486" title="Special:BookSources/978-1581133486"><bdi>978-1581133486</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:10316730">10316730</a>. Archived from <a rel="nofollow" class="external text" href="http://www10.org/cdrom/papers/210/index.html">the original</a> on 25 June 2014<span class="reference-accessdate">. Retrieved <span class="nowrap">25 January</span> 2007</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=An+adaptive+model+for+optimizing+performance+of+an+incremental+web+crawler&rft.btitle=Proceedings+of+the+10th+international+conference+on+World+Wide+Web&rft.pages=106-113&rft.date=2001&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.1018.1506%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A10316730%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F371920.371960&rft.isbn=978-1581133486&rft.au=Edwards%2C+J.%3B+McCurley%2C+K.+S.%3B+and+Tomlin%2C+J.+A.&rft_id=http%3A%2F%2Fwww10.org%2Fcdrom%2Fpapers%2F210%2Findex.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span><span class="cs1-maint citation-comment"><code class="cs1-code">{{<a href="/wiki/Template:Cite_book" title="Template:Cite book">cite book</a>}}</code>: CS1 maint: multiple names: authors list (<a href="/wiki/Category:CS1_maint:_multiple_names:_authors_list" title="Category:CS1 maint: multiple names: authors list">link</a>)</span></span> </li> <li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCastillo2004" class="citation thesis cs1">Castillo, Carlos (2004). <a rel="nofollow" class="external text" href="http://chato.cl/research/crawling_thesis"><i>Effective Web Crawling</i></a> (PhD thesis). University of Chile<span class="reference-accessdate">. Retrieved <span class="nowrap">3 August</span> 2010</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Adissertation&rft.title=Effective+Web+Crawling&rft.degree=PhD&rft.inst=University+of+Chile&rft.date=2004&rft.aulast=Castillo&rft.aufirst=Carlos&rft_id=http%3A%2F%2Fchato.cl%2Fresearch%2Fcrawling_thesis&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFGulls,_A.A._Signori2005" class="citation conference cs1">Gulls, A.; A. Signori (2005). "The indexable web is more than 11.5 billion pages". <i>Special interest tracks and posters of the 14th international conference on World Wide Web</i>. ACM Press. pp. 902–903. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F1062745.1062789">10.1145/1062745.1062789</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=The+indexable+web+is+more+than+11.5+billion+pages&rft.btitle=Special+interest+tracks+and+posters+of+the+14th+international+conference+on+World+Wide+Web&rft.pages=902-903&rft.pub=ACM+Press&rft.date=2005&rft_id=info%3Adoi%2F10.1145%2F1062745.1062789&rft.au=Gulls%2C+A.&rft.au=A.+Signori&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFLawrence,_SteveC._Lee_Giles1999" class="citation journal cs1">Lawrence, Steve; C. Lee Giles (8 July 1999). <a rel="nofollow" class="external text" href="https://doi.org/10.1038%2F21987">"Accessibility of information on the web"</a>. <i>Nature</i>. <b>400</b> (6740): 107–9. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/1999Natur.400..107L">1999Natur.400..107L</a>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://doi.org/10.1038%2F21987">10.1038/21987</a></span>. <a href="/wiki/PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/10428673">10428673</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:4347646">4347646</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Nature&rft.atitle=Accessibility+of+information+on+the+web&rft.volume=400&rft.issue=6740&rft.pages=107-9&rft.date=1999-07-08&rft_id=info%3Adoi%2F10.1038%2F21987&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A4347646%23id-name%3DS2CID&rft_id=info%3Apmid%2F10428673&rft_id=info%3Abibcode%2F1999Natur.400..107L&rft.au=Lawrence%2C+Steve&rft.au=C.+Lee+Giles&rft_id=https%3A%2F%2Fdoi.org%2F10.1038%252F21987&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-10"><span class="mw-cite-backlink"><b><a href="#cite_ref-10">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCho,_J.Garcia-Molina,_H.Page,_L.1998" class="citation journal cs1">Cho, J.; Garcia-Molina, H.; Page, L. (April 1998). <a rel="nofollow" class="external text" href="http://ilpubs.stanford.edu:8090/347/">"Efficient Crawling Through URL Ordering"</a>. <i>Seventh International World-Wide Web Conference</i>. Brisbane, Australia. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1142%2F3725">10.1142/3725</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-981-02-3400-3" title="Special:BookSources/978-981-02-3400-3"><bdi>978-981-02-3400-3</bdi></a><span class="reference-accessdate">. Retrieved <span class="nowrap">23 March</span> 2009</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Seventh+International+World-Wide+Web+Conference&rft.atitle=Efficient+Crawling+Through+URL+Ordering&rft.date=1998-04&rft_id=info%3Adoi%2F10.1142%2F3725&rft.isbn=978-981-02-3400-3&rft.au=Cho%2C+J.&rft.au=Garcia-Molina%2C+H.&rft.au=Page%2C+L.&rft_id=http%3A%2F%2Filpubs.stanford.edu%3A8090%2F347%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-11"><span class="mw-cite-backlink"><b><a href="#cite_ref-11">^</a></b></span> <span class="reference-text">Cho, Junghoo, <a rel="nofollow" class="external text" href="http://oak.cs.ucla.edu/~cho/papers/cho-thesis.pdf">"Crawling the Web: Discovery and Maintenance of a Large-Scale Web Data"</a>, PhD dissertation, Department of Computer Science, Stanford University, November 2001.</span> </li> <li id="cite_note-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-12">^</a></b></span> <span class="reference-text">Najork, Marc and Janet L. Wiener. <a rel="nofollow" class="external text" href="http://www10.org/cdrom/papers/pdf/p208.pdf">"Breadth-first crawling yields high-quality pages".</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20171224210412/http://www10.org/cdrom/papers/pdf/p208.pdf">Archived</a> 24 December 2017 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a> In: <i>Proceedings of the Tenth Conference on World Wide Web</i>, pages 114–118, Hong Kong, May 2001. Elsevier Science.</span> </li> <li id="cite_note-13"><span class="mw-cite-backlink"><b><a href="#cite_ref-13">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFAbiteboul,_SergeMihai_PredaGregory_Cobena2003" class="citation conference cs1">Abiteboul, Serge; Mihai Preda; Gregory Cobena (2003). <a rel="nofollow" class="external text" href="http://www2003.org/cdrom/papers/refereed/p007/p7-abiteboul.html">"Adaptive on-line page importance computation"</a>. <i>Proceedings of the 12th international conference on World Wide Web</i>. Budapest, Hungary: ACM. pp. 280–290. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F775152.775192">10.1145/775152.775192</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/1-58113-680-3" title="Special:BookSources/1-58113-680-3"><bdi>1-58113-680-3</bdi></a><span class="reference-accessdate">. Retrieved <span class="nowrap">22 March</span> 2009</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Adaptive+on-line+page+importance+computation&rft.btitle=Proceedings+of+the+12th+international+conference+on+World+Wide+Web&rft.place=Budapest%2C+Hungary&rft.pages=280-290&rft.pub=ACM&rft.date=2003&rft_id=info%3Adoi%2F10.1145%2F775152.775192&rft.isbn=1-58113-680-3&rft.au=Abiteboul%2C+Serge&rft.au=Mihai+Preda&rft.au=Gregory+Cobena&rft_id=http%3A%2F%2Fwww2003.org%2Fcdrom%2Fpapers%2Frefereed%2Fp007%2Fp7-abiteboul.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-14"><span class="mw-cite-backlink"><b><a href="#cite_ref-14">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBoldi,_PaoloBruno_CodenottiMassimo_SantiniSebastiano_Vigna2004" class="citation journal cs1">Boldi, Paolo; Bruno Codenotti; Massimo Santini; Sebastiano Vigna (2004). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20090320030833/http://vigna.dsi.unimi.it/ftp/papers/UbiCrawler.pdf">"UbiCrawler: a scalable fully distributed Web crawler"</a> <span class="cs1-format">(PDF)</span>. <i>Software: Practice and Experience</i>. <b>34</b> (8): 711–726. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.2.5538">10.1.1.2.5538</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fspe.587">10.1002/spe.587</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:325714">325714</a>. Archived from <a rel="nofollow" class="external text" href="http://vigna.dsi.unimi.it/ftp/papers/UbiCrawler.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 20 March 2009<span class="reference-accessdate">. Retrieved <span class="nowrap">23 March</span> 2009</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Software%3A+Practice+and+Experience&rft.atitle=UbiCrawler%3A+a+scalable+fully+distributed+Web+crawler&rft.volume=34&rft.issue=8&rft.pages=711-726&rft.date=2004&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.2.5538%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A325714%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1002%2Fspe.587&rft.au=Boldi%2C+Paolo&rft.au=Bruno+Codenotti&rft.au=Massimo+Santini&rft.au=Sebastiano+Vigna&rft_id=http%3A%2F%2Fvigna.dsi.unimi.it%2Fftp%2Fpapers%2FUbiCrawler.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-15"><span class="mw-cite-backlink"><b><a href="#cite_ref-15">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBoldi,_PaoloMassimo_SantiniSebastiano_Vigna2004" class="citation book cs1">Boldi, Paolo; Massimo Santini; Sebastiano Vigna (2004). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20051001063017/http://vigna.dsi.unimi.it/ftp/papers/ParadoxicalPageRank.pdf">"Do Your Worst to Make the Best: Paradoxical Effects in PageRank Incremental Computations"</a> <span class="cs1-format">(PDF)</span>. <i>Algorithms and Models for the Web-Graph</i>. Lecture Notes in Computer Science. Vol. 3243. pp. 168–180. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-540-30216-2_14">10.1007/978-3-540-30216-2_14</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-23427-2" title="Special:BookSources/978-3-540-23427-2"><bdi>978-3-540-23427-2</bdi></a>. Archived from <a rel="nofollow" class="external text" href="http://vigna.dsi.unimi.it/ftp/papers/ParadoxicalPageRank.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 1 October 2005<span class="reference-accessdate">. Retrieved <span class="nowrap">23 March</span> 2009</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Do+Your+Worst+to+Make+the+Best%3A+Paradoxical+Effects+in+PageRank+Incremental+Computations&rft.btitle=Algorithms+and+Models+for+the+Web-Graph&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=168-180&rft.date=2004&rft_id=info%3Adoi%2F10.1007%2F978-3-540-30216-2_14&rft.isbn=978-3-540-23427-2&rft.au=Boldi%2C+Paolo&rft.au=Massimo+Santini&rft.au=Sebastiano+Vigna&rft_id=http%3A%2F%2Fvigna.dsi.unimi.it%2Fftp%2Fpapers%2FParadoxicalPageRank.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-baeza2005-16"><span class="mw-cite-backlink"><b><a href="#cite_ref-baeza2005_16-0">^</a></b></span> <span class="reference-text">Baeza-Yates, R.; Castillo, C.; Marin, M. and Rodriguez, A. (2005). <a rel="nofollow" class="external text" href="http://chato.cl/papers/baeza05_crawling_country_better_breadth_first_web_page_ordering.pdf">"Crawling a Country: Better Strategies than Breadth-First for Web Page Ordering."</a> In: <i>Proceedings of the Industrial and Practical Experience track of the 14th conference on World Wide Web</i>, pages 864–872, Chiba, Japan. ACM Press.</span> </li> <li id="cite_note-17"><span class="mw-cite-backlink"><b><a href="#cite_ref-17">^</a></b></span> <span class="reference-text">Shervin Daneshpajouh, Mojtaba Mohammadi Nasiri, Mohammad Ghodsi, <a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/220724572_A_Fast_Community_Based_Algorithm_for_Generating_Web_Crawler_Seeds_Set">A Fast Community Based Algorithm for Generating Crawler Seeds Set</a>. In: <i>Proceedings of 4th International Conference on Web Information Systems and Technologies</i> (<i><a href="/wiki/Webist" title="Webist">Webist</a></i>-2008), Funchal, Portugal, May 2008.</span> </li> <li id="cite_note-18"><span class="mw-cite-backlink"><b><a href="#cite_ref-18">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFPantSrinivasanMenczer2004" class="citation book cs1">Pant, Gautam; Srinivasan, Padmini; Menczer, Filippo (2004). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20090320030832/http://dollar.biz.uiowa.edu/~pant/Papers/crawling.pdf">"Crawling the Web"</a> <span class="cs1-format">(PDF)</span>. In Levene, Mark; <a href="/wiki/Alexandra_Poulovassilis" title="Alexandra Poulovassilis">Poulovassilis, Alexandra</a> (eds.). <i>Web Dynamics: Adapting to Change in Content, Size, Topology and Use</i>. Springer. pp. 153–178. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-540-40676-1" title="Special:BookSources/978-3-540-40676-1"><bdi>978-3-540-40676-1</bdi></a>. Archived from <a rel="nofollow" class="external text" href="http://dollar.biz.uiowa.edu/~pant/Papers/crawling.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 20 March 2009<span class="reference-accessdate">. Retrieved <span class="nowrap">9 May</span> 2006</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Crawling+the+Web&rft.btitle=Web+Dynamics%3A+Adapting+to+Change+in+Content%2C+Size%2C+Topology+and+Use&rft.pages=153-178&rft.pub=Springer&rft.date=2004&rft.isbn=978-3-540-40676-1&rft.aulast=Pant&rft.aufirst=Gautam&rft.au=Srinivasan%2C+Padmini&rft.au=Menczer%2C+Filippo&rft_id=http%3A%2F%2Fdollar.biz.uiowa.edu%2F~pant%2FPapers%2Fcrawling.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-19"><span class="mw-cite-backlink"><b><a href="#cite_ref-19">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCothey2004" class="citation journal cs1">Cothey, Viv (2004). <a rel="nofollow" class="external text" href="http://www.scit.wlv.ac.uk/~in7803/publications/cothey_2004.pdf">"Web-crawling reliability"</a> <span class="cs1-format">(PDF)</span>. <i>Journal of the American Society for Information Science and Technology</i>. <b>55</b> (14): 1228–1238. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.117.185">10.1.1.117.185</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fasi.20078">10.1002/asi.20078</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+American+Society+for+Information+Science+and+Technology&rft.atitle=Web-crawling+reliability&rft.volume=55&rft.issue=14&rft.pages=1228-1238&rft.date=2004&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.117.185%23id-name%3DCiteSeerX&rft_id=info%3Adoi%2F10.1002%2Fasi.20078&rft.aulast=Cothey&rft.aufirst=Viv&rft_id=http%3A%2F%2Fwww.scit.wlv.ac.uk%2F~in7803%2Fpublications%2Fcothey_2004.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-20"><span class="mw-cite-backlink"><b><a href="#cite_ref-20">^</a></b></span> <span class="reference-text">Menczer, F. (1997). <a rel="nofollow" class="external text" href="http://informatics.indiana.edu/fil/Papers/ICML.ps">ARACHNID: Adaptive Retrieval Agents Choosing Heuristic Neighborhoods for Information Discovery</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20121221113620/http://informatics.indiana.edu/fil/Papers/ICML.ps">Archived</a> 21 December 2012 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>. In D. Fisher, ed., Machine Learning: Proceedings of the 14th International Conference (ICML97). Morgan Kaufmann</span> </li> <li id="cite_note-21"><span class="mw-cite-backlink"><b><a href="#cite_ref-21">^</a></b></span> <span class="reference-text">Menczer, F. and Belew, R.K. (1998). <a rel="nofollow" class="external text" href="http://informatics.indiana.edu/fil/Papers/AA98.ps">Adaptive Information Agents in Distributed Textual Environments</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20121221113630/http://informatics.indiana.edu/fil/Papers/AA98.ps">Archived</a> 21 December 2012 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>. In K. Sycara and M. Wooldridge (eds.) Proc. 2nd Intl. Conf. on Autonomous Agents (Agents '98). ACM Press</span> </li> <li id="cite_note-22"><span class="mw-cite-backlink"><b><a href="#cite_ref-22">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChakrabartiVan_Den_BergDom1999" class="citation journal cs1">Chakrabarti, Soumen; Van Den Berg, Martin; Dom, Byron (1999). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20040317210216/http://www.fxpal.com/people/vdberg/pubs/www8/www1999f.pdf">"Focused crawling: A new approach to topic-specific Web resource discovery"</a> <span class="cs1-format">(PDF)</span>. <i>Computer Networks</i>. <b>31</b> (11–16): 1623–1640. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fs1389-1286%2899%2900052-3">10.1016/s1389-1286(99)00052-3</a>. Archived from <a rel="nofollow" class="external text" href="http://www.fxpal.com/people/vdberg/pubs/www8/www1999f.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 17 March 2004.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Computer+Networks&rft.atitle=Focused+crawling%3A+A+new+approach+to+topic-specific+Web+resource+discovery&rft.volume=31&rft.issue=11%E2%80%9316&rft.pages=1623-1640&rft.date=1999&rft_id=info%3Adoi%2F10.1016%2Fs1389-1286%2899%2900052-3&rft.aulast=Chakrabarti&rft.aufirst=Soumen&rft.au=Van+Den+Berg%2C+Martin&rft.au=Dom%2C+Byron&rft_id=http%3A%2F%2Fwww.fxpal.com%2Fpeople%2Fvdberg%2Fpubs%2Fwww8%2Fwww1999f.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-pinkerton1994-23"><span class="mw-cite-backlink"><b><a href="#cite_ref-pinkerton1994_23-0">^</a></b></span> <span class="reference-text">Pinkerton, B. (1994). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20010904075500/http://archive.ncsa.uiuc.edu/SDG/IT94/Proceedings/Searching/pinkerton/WebCrawler.html">Finding what people want: Experiences with the WebCrawler</a>. In Proceedings of the First World Wide Web Conference, Geneva, Switzerland.</span> </li> <li id="cite_note-24"><span class="mw-cite-backlink"><b><a href="#cite_ref-24">^</a></b></span> <span class="reference-text">Diligenti, M., Coetzee, F., Lawrence, S., Giles, C. L., and Gori, M. (2000). <a rel="nofollow" class="external text" href="http://clgiles.ist.psu.edu/papers/VLDB-2000-focused-crawling.pdf">Focused crawling using context graphs</a>. In Proceedings of 26th International Conference on Very Large Databases (VLDB), pages 527-534, Cairo, Egypt.</span> </li> <li id="cite_note-25"><span class="mw-cite-backlink"><b><a href="#cite_ref-25">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWuTeregowdaKhabsaCarman2012" class="citation book cs1">Wu, Jian; Teregowda, Pradeep; Khabsa, Madian; Carman, Stephen; Jordan, Douglas; San Pedro Wandelmer, Jose; Lu, Xin; Mitra, Prasenjit; Giles, C. Lee (2012). "Web crawler middleware for search engine digital libraries". <i>Proceedings of the twelfth international workshop on Web information and data management - WIDM '12</i>. p. 57. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F2389936.2389949">10.1145/2389936.2389949</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781450317207" title="Special:BookSources/9781450317207"><bdi>9781450317207</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:18513666">18513666</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=Web+crawler+middleware+for+search+engine+digital+libraries&rft.btitle=Proceedings+of+the+twelfth+international+workshop+on+Web+information+and+data+management+-+WIDM+%2712&rft.pages=57&rft.date=2012&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A18513666%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F2389936.2389949&rft.isbn=9781450317207&rft.aulast=Wu&rft.aufirst=Jian&rft.au=Teregowda%2C+Pradeep&rft.au=Khabsa%2C+Madian&rft.au=Carman%2C+Stephen&rft.au=Jordan%2C+Douglas&rft.au=San+Pedro+Wandelmer%2C+Jose&rft.au=Lu%2C+Xin&rft.au=Mitra%2C+Prasenjit&rft.au=Giles%2C+C.+Lee&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-26"><span class="mw-cite-backlink"><b><a href="#cite_ref-26">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFWuTeregowdaRamírezMitra2012" class="citation book cs1">Wu, Jian; Teregowda, Pradeep; Ramírez, Juan Pablo Fernández; Mitra, Prasenjit; Zheng, Shuyi; Giles, C. Lee (2012). "The evolution of a crawling strategy for an academic document search engine". <i>Proceedings of the 3rd Annual ACM Web Science Conference on - Web </i>Sci<i> '12</i>. pp. 340–343. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F2380718.2380762">10.1145/2380718.2380762</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/9781450312288" title="Special:BookSources/9781450312288"><bdi>9781450312288</bdi></a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:16718130">16718130</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=The+evolution+of+a+crawling+strategy+for+an+academic+document+search+engine&rft.btitle=Proceedings+of+the+3rd+Annual+ACM+Web+Science+Conference+on+-+Web+Sci+%2712&rft.pages=340-343&rft.date=2012&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A16718130%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F2380718.2380762&rft.isbn=9781450312288&rft.aulast=Wu&rft.aufirst=Jian&rft.au=Teregowda%2C+Pradeep&rft.au=Ram%C3%ADrez%2C+Juan+Pablo+Fern%C3%A1ndez&rft.au=Mitra%2C+Prasenjit&rft.au=Zheng%2C+Shuyi&rft.au=Giles%2C+C.+Lee&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-27"><span class="mw-cite-backlink"><b><a href="#cite_ref-27">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDongHussainChang2009" class="citation book cs1">Dong, Hai; Hussain, Farookh Khadeer; Chang, Elizabeth (2009). <a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/44241179">"State of the Art in Semantic Focused Crawlers"</a>. <i>Computational Science and Its Applications – ICCSA 2009</i>. Lecture Notes in Computer Science. Vol. 5593. pp. 910–924. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-642-02457-3_74">10.1007/978-3-642-02457-3_74</a>. <a href="/wiki/Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/20.500.11937%2F48288">20.500.11937/48288</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/978-3-642-02456-6" title="Special:BookSources/978-3-642-02456-6"><bdi>978-3-642-02456-6</bdi></a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=bookitem&rft.atitle=State+of+the+Art+in+Semantic+Focused+Crawlers&rft.btitle=Computational+Science+and+Its+Applications+%E2%80%93+ICCSA+2009&rft.series=Lecture+Notes+in+Computer+Science&rft.pages=910-924&rft.date=2009&rft_id=info%3Ahdl%2F20.500.11937%2F48288&rft_id=info%3Adoi%2F10.1007%2F978-3-642-02457-3_74&rft.isbn=978-3-642-02456-6&rft.aulast=Dong&rft.aufirst=Hai&rft.au=Hussain%2C+Farookh+Khadeer&rft.au=Chang%2C+Elizabeth&rft_id=https%3A%2F%2Fwww.researchgate.net%2Fpublication%2F44241179&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-28"><span class="mw-cite-backlink"><b><a href="#cite_ref-28">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDongHussain2013" class="citation journal cs1">Dong, Hai; Hussain, Farookh Khadeer (2013). <a rel="nofollow" class="external text" href="https://www.researchgate.net/publication/264620349">"SOF: A semi-supervised ontology-learning-based focused crawler"</a>. <i>Concurrency and Computation: Practice and Experience</i>. <b>25</b> (12): 1755–1770. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fcpe.2980">10.1002/cpe.2980</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:205690364">205690364</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Concurrency+and+Computation%3A+Practice+and+Experience&rft.atitle=SOF%3A+A+semi-supervised+ontology-learning-based+focused+crawler&rft.volume=25&rft.issue=12&rft.pages=1755-1770&rft.date=2013&rft_id=info%3Adoi%2F10.1002%2Fcpe.2980&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A205690364%23id-name%3DS2CID&rft.aulast=Dong&rft.aufirst=Hai&rft.au=Hussain%2C+Farookh+Khadeer&rft_id=https%3A%2F%2Fwww.researchgate.net%2Fpublication%2F264620349&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-29"><span class="mw-cite-backlink"><b><a href="#cite_ref-29">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJunghoo_ChoHector_Garcia-Molina2000" class="citation conference cs1">Junghoo Cho; Hector Garcia-Molina (2000). <a rel="nofollow" class="external text" href="http://www.cs.brown.edu/courses/cs227/2002/cache/Cho.pdf">"Synchronizing a database to improve freshness"</a> <span class="cs1-format">(PDF)</span>. <i>Proceedings of the 2000 ACM SIGMOD international conference on Management of data</i>. Dallas, Texas, United States: ACM. pp. 117–128. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F342009.335391">10.1145/342009.335391</a>. <a href="/wiki/ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <a href="/wiki/Special:BookSources/1-58113-217-4" title="Special:BookSources/1-58113-217-4"><bdi>1-58113-217-4</bdi></a><span class="reference-accessdate">. Retrieved <span class="nowrap">23 March</span> 2009</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=conference&rft.atitle=Synchronizing+a+database+to+improve+freshness&rft.btitle=Proceedings+of+the+2000+ACM+SIGMOD+international+conference+on+Management+of+data&rft.place=Dallas%2C+Texas%2C+United+States&rft.pages=117-128&rft.pub=ACM&rft.date=2000&rft_id=info%3Adoi%2F10.1145%2F342009.335391&rft.isbn=1-58113-217-4&rft.au=Junghoo+Cho&rft.au=Hector+Garcia-Molina&rft_id=http%3A%2F%2Fwww.cs.brown.edu%2Fcourses%2Fcs227%2F2002%2Fcache%2FCho.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-coffman-30"><span class="mw-cite-backlink">^ <a href="#cite_ref-coffman_30-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-coffman_30-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFE._G._Coffman_JrZhen_LiuRichard_R._Weber1998" class="citation journal cs1">E. G. Coffman Jr; Zhen Liu; Richard R. Weber (1998). "Optimal robot scheduling for Web search engines". <i>Journal of Scheduling</i>. <b>1</b> (1): 15–29. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.36.6087">10.1.1.36.6087</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2F%28SICI%291099-1425%28199806%291%3A1%3C15%3A%3AAID-JOS3%3E3.0.CO%3B2-K">10.1002/(SICI)1099-1425(199806)1:1<15::AID-JOS3>3.0.CO;2-K</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+Scheduling&rft.atitle=Optimal+robot+scheduling+for+Web+search+engines&rft.volume=1&rft.issue=1&rft.pages=15-29&rft.date=1998&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.36.6087%23id-name%3DCiteSeerX&rft_id=info%3Adoi%2F10.1002%2F%28SICI%291099-1425%28199806%291%3A1%3C15%3A%3AAID-JOS3%3E3.0.CO%3B2-K&rft.au=E.+G.+Coffman+Jr&rft.au=Zhen+Liu&rft.au=Richard+R.+Weber&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-cho2003-31"><span class="mw-cite-backlink">^ <a href="#cite_ref-cho2003_31-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-cho2003_31-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFChoGarcia-Molina2003" class="citation journal cs1">Cho, Junghoo; Garcia-Molina, Hector (2003). "Effective page refresh policies for Web crawlers". <i>ACM Transactions on Database Systems</i>. <b>28</b> (4): 390–426. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F958942.958945">10.1145/958942.958945</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:147958">147958</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=ACM+Transactions+on+Database+Systems&rft.atitle=Effective+page+refresh+policies+for+Web+crawlers&rft.volume=28&rft.issue=4&rft.pages=390-426&rft.date=2003&rft_id=info%3Adoi%2F10.1145%2F958942.958945&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A147958%23id-name%3DS2CID&rft.aulast=Cho&rft.aufirst=Junghoo&rft.au=Garcia-Molina%2C+Hector&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-cho2003a-32"><span class="mw-cite-backlink">^ <a href="#cite_ref-cho2003a_32-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-cho2003a_32-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFJunghoo_ChoHector_Garcia-Molina2003" class="citation journal cs1">Junghoo Cho; Hector Garcia-Molina (2003). "Estimating frequency of change". <i>ACM Transactions on Internet Technology</i>. <b>3</b> (3): 256–290. <a href="/wiki/CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.59.5877">10.1.1.59.5877</a></span>. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F857166.857170">10.1145/857166.857170</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:9362566">9362566</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=ACM+Transactions+on+Internet+Technology&rft.atitle=Estimating+frequency+of+change&rft.volume=3&rft.issue=3&rft.pages=256-290&rft.date=2003&rft_id=https%3A%2F%2Fciteseerx.ist.psu.edu%2Fviewdoc%2Fsummary%3Fdoi%3D10.1.1.59.5877%23id-name%3DCiteSeerX&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A9362566%23id-name%3DS2CID&rft_id=info%3Adoi%2F10.1145%2F857166.857170&rft.au=Junghoo+Cho&rft.au=Hector+Garcia-Molina&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-33"><span class="mw-cite-backlink"><b><a href="#cite_ref-33">^</a></b></span> <span class="reference-text">Ipeirotis, P., Ntoulas, A., Cho, J., Gravano, L. (2005) <a rel="nofollow" class="external text" href="http://pages.stern.nyu.edu/~panos/publications/icde2005.pdf">Modeling and managing content changes in text databases</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20050905013119/http://pages.stern.nyu.edu/~panos/publications/icde2005.pdf">Archived</a> 5 September 2005 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>. In Proceedings of the 21st IEEE International Conference on Data Engineering, pages 606-617, April 2005, Tokyo.</span> </li> <li id="cite_note-34"><span class="mw-cite-backlink"><b><a href="#cite_ref-34">^</a></b></span> <span class="reference-text">Koster, M. (1995). Robots in the web: threat or treat? ConneXions, 9(4).</span> </li> <li id="cite_note-35"><span class="mw-cite-backlink"><b><a href="#cite_ref-35">^</a></b></span> <span class="reference-text">Koster, M. (1996). <a rel="nofollow" class="external text" href="http://www.robotstxt.org/wc/exclusion.html">A standard for robot exclusion</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20071107021800/http://www.robotstxt.org/wc/exclusion.html">Archived</a> 7 November 2007 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>.</span> </li> <li id="cite_note-36"><span class="mw-cite-backlink"><b><a href="#cite_ref-36">^</a></b></span> <span class="reference-text">Koster, M. (1993). <a rel="nofollow" class="external text" href="http://www.robotstxt.org/wc/guidelines.html">Guidelines for robots writers</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20050422045839/http://www.robotstxt.org/wc/guidelines.html">Archived</a> 22 April 2005 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>.</span> </li> <li id="cite_note-baeza2002-37"><span class="mw-cite-backlink"><b><a href="#cite_ref-baeza2002_37-0">^</a></b></span> <span class="reference-text">Baeza-Yates, R. and Castillo, C. (2002). <a rel="nofollow" class="external text" href="http://www.chato.cl/papers/baeza02balancing.pdf">Balancing volume, quality and freshness in Web crawling</a>. In Soft Computing Systems – Design, Management and Applications, pages 565–572, Santiago, Chile. IOS Press Amsterdam.</span> </li> <li id="cite_note-38"><span class="mw-cite-backlink"><b><a href="#cite_ref-38">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFHeydon,_AllanNajork,_Marc1999" class="citation journal cs1">Heydon, Allan; Najork, Marc (26 June 1999). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20060219085958/http://www.cindoc.csic.es/cybermetrics/pdf/68.pdf">"Mercator: A Scalable, Extensible Web Crawler"</a> <span class="cs1-format">(PDF)</span>. Archived from <a rel="nofollow" class="external text" href="http://www.cindoc.csic.es/cybermetrics/pdf/68.pdf">the original</a> <span class="cs1-format">(PDF)</span> on 19 February 2006<span class="reference-accessdate">. Retrieved <span class="nowrap">22 March</span> 2009</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.atitle=Mercator%3A+A+Scalable%2C+Extensible+Web+Crawler&rft.date=1999-06-26&rft.au=Heydon%2C+Allan&rft.au=Najork%2C+Marc&rft_id=http%3A%2F%2Fwww.cindoc.csic.es%2Fcybermetrics%2Fpdf%2F68.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span> <span class="cs1-visible-error citation-comment"><code class="cs1-code">{{<a href="/wiki/Template:Cite_journal" title="Template:Cite journal">cite journal</a>}}</code>: </span><span class="cs1-visible-error citation-comment">Cite journal requires <code class="cs1-code">|journal=</code> (<a href="/wiki/Help:CS1_errors#missing_periodical" title="Help:CS1 errors">help</a>)</span></span> </li> <li id="cite_note-39"><span class="mw-cite-backlink"><b><a href="#cite_ref-39">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFDillKumarMccurleyRajagopalan2002" class="citation journal cs1">Dill, S.; Kumar, R.; Mccurley, K. S.; Rajagopalan, S.; Sivakumar, D.; Tomkins, A. (2002). <a rel="nofollow" class="external text" href="http://www.mccurley.org/papers/fractal.pdf">"Self-similarity in the web"</a> <span class="cs1-format">(PDF)</span>. <i>ACM Transactions on Internet Technology</i>. <b>2</b> (3): 205–223. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1145%2F572326.572328">10.1145/572326.572328</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:6416041">6416041</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=ACM+Transactions+on+Internet+Technology&rft.atitle=Self-similarity+in+the+web&rft.volume=2&rft.issue=3&rft.pages=205-223&rft.date=2002&rft_id=info%3Adoi%2F10.1145%2F572326.572328&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A6416041%23id-name%3DS2CID&rft.aulast=Dill&rft.aufirst=S.&rft.au=Kumar%2C+R.&rft.au=Mccurley%2C+K.+S.&rft.au=Rajagopalan%2C+S.&rft.au=Sivakumar%2C+D.&rft.au=Tomkins%2C+A.&rft_id=http%3A%2F%2Fwww.mccurley.org%2Fpapers%2Ffractal.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-40"><span class="mw-cite-backlink"><b><a href="#cite_ref-40">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFM._ThelwallD._Stuart2006" class="citation journal cs1">M. Thelwall; D. Stuart (2006). <a rel="nofollow" class="external text" href="http://www.scit.wlv.ac.uk/%7Ecm1993/papers/Web_Crawling_Ethics_preprint.doc">"Web crawling ethics revisited: Cost, privacy and denial of service"</a>. <i>Journal of the American Society for Information Science and Technology</i>. <b>57</b> (13): 1771–1779. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fasi.20388">10.1002/asi.20388</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Journal+of+the+American+Society+for+Information+Science+and+Technology&rft.atitle=Web+crawling+ethics+revisited%3A+Cost%2C+privacy+and+denial+of+service&rft.volume=57&rft.issue=13&rft.pages=1771-1779&rft.date=2006&rft_id=info%3Adoi%2F10.1002%2Fasi.20388&rft.au=M.+Thelwall&rft.au=D.+Stuart&rft_id=http%3A%2F%2Fwww.scit.wlv.ac.uk%2F%257Ecm1993%2Fpapers%2FWeb_Crawling_Ethics_preprint.doc&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-brin1998-41"><span class="mw-cite-backlink"><b><a href="#cite_ref-brin1998_41-0">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFBrinPage1998" class="citation journal cs1">Brin, Sergey; Page, Lawrence (1998). <a rel="nofollow" class="external text" href="http://infolab.stanford.edu/~backrub/google.html">"The anatomy of a large-scale hypertextual Web search engine"</a>. <i>Computer Networks and ISDN Systems</i>. <b>30</b> (1–7): 107–117. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fs0169-7552%2898%2900110-x">10.1016/s0169-7552(98)00110-x</a>. <a href="/wiki/S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:7587743">7587743</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Computer+Networks+and+ISDN+Systems&rft.atitle=The+anatomy+of+a+large-scale+hypertextual+Web+search+engine&rft.volume=30&rft.issue=1%E2%80%937&rft.pages=107-117&rft.date=1998&rft_id=info%3Adoi%2F10.1016%2Fs0169-7552%2898%2900110-x&rft_id=https%3A%2F%2Fapi.semanticscholar.org%2FCorpusID%3A7587743%23id-name%3DS2CID&rft.aulast=Brin&rft.aufirst=Sergey&rft.au=Page%2C+Lawrence&rft_id=http%3A%2F%2Finfolab.stanford.edu%2F~backrub%2Fgoogle.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-shkapenyuk2002-42"><span class="mw-cite-backlink"><b><a href="#cite_ref-shkapenyuk2002_42-0">^</a></b></span> <span class="reference-text">Shkapenyuk, V. and <a href="/wiki/Torsten_Suel" title="Torsten Suel">Suel, T.</a> (2002). <a rel="nofollow" class="external text" href="http://cis.poly.edu/tr/tr-cis-2001-03.pdf">Design and implementation of a high performance distributed web crawler</a>. In Proceedings of the 18th International Conference on Data Engineering (ICDE), pages 357-368, San Jose, California. IEEE CS Press.</span> </li> <li id="cite_note-43"><span class="mw-cite-backlink"><b><a href="#cite_ref-43">^</a></b></span> <span class="reference-text">Shestakov, Denis (2008). <a rel="nofollow" class="external text" href="https://oa.doria.fi/handle/10024/38506"><i>Search Interfaces on the Web: Querying and Characterizing</i></a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20140706120641/https://oa.doria.fi/handle/10024/38506">Archived</a> 6 July 2014 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>. TUCS Doctoral Dissertations 104, University of Turku</span> </li> <li id="cite_note-44"><span class="mw-cite-backlink"><b><a href="#cite_ref-44">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFMichael_L_NelsonHerbert_Van_de_SompelXiaoming_LiuTerry_L_Harrison2005" class="citation journal cs1">Michael L Nelson; Herbert Van de Sompel; Xiaoming Liu; Terry L Harrison; Nathan McFarland (24 March 2005). "mod_oai: An Apache Module for Metadata Harvesting": cs/0503069. <a href="/wiki/ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/cs/0503069">cs/0503069</a></span>. <a href="/wiki/Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2005cs........3069N">2005cs........3069N</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.atitle=mod_oai%3A+An+Apache+Module+for+Metadata+Harvesting&rft.pages=cs%2F0503069&rft.date=2005-03-24&rft_id=info%3Aarxiv%2Fcs%2F0503069&rft_id=info%3Abibcode%2F2005cs........3069N&rft.au=Michael+L+Nelson&rft.au=Herbert+Van+de+Sompel&rft.au=Xiaoming+Liu&rft.au=Terry+L+Harrison&rft.au=Nathan+McFarland&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span> <span class="cs1-visible-error citation-comment"><code class="cs1-code">{{<a href="/wiki/Template:Cite_journal" title="Template:Cite journal">cite journal</a>}}</code>: </span><span class="cs1-visible-error citation-comment">Cite journal requires <code class="cs1-code">|journal=</code> (<a href="/wiki/Help:CS1_errors#missing_periodical" title="Help:CS1 errors">help</a>)</span></span> </li> <li id="cite_note-45"><span class="mw-cite-backlink"><b><a href="#cite_ref-45">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFShestakovBhowmickLim2005" class="citation journal cs1">Shestakov, Denis; Bhowmick, Sourav S.; Lim, Ee-Peng (2005). <a rel="nofollow" class="external text" href="http://www.mendeley.com/download/public/1423991/3893295922/dc0f7d824fd2a8fbbc84f6fdf9e4f337d343987d/dl.pdf">"DEQUE: Querying the Deep Web"</a> <span class="cs1-format">(PDF)</span>. <i>Data & Knowledge Engineering</i>. <b>52</b> (3): 273–311. <a href="/wiki/Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fs0169-023x%2804%2900107-7">10.1016/s0169-023x(04)00107-7</a>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Data+%26+Knowledge+Engineering&rft.atitle=DEQUE%3A+Querying+the+Deep+Web&rft.volume=52&rft.issue=3&rft.pages=273-311&rft.date=2005&rft_id=info%3Adoi%2F10.1016%2Fs0169-023x%2804%2900107-7&rft.aulast=Shestakov&rft.aufirst=Denis&rft.au=Bhowmick%2C+Sourav+S.&rft.au=Lim%2C+Ee-Peng&rft_id=http%3A%2F%2Fwww.mendeley.com%2Fdownload%2Fpublic%2F1423991%2F3893295922%2Fdc0f7d824fd2a8fbbc84f6fdf9e4f337d343987d%2Fdl.pdf&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-46"><span class="mw-cite-backlink"><b><a href="#cite_ref-46">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://support.google.com/webmasters/bin/answer.py?hl=en&answer=174992">"AJAX crawling: Guide for webmasters and developers"</a><span class="reference-accessdate">. Retrieved <span class="nowrap">17 March</span> 2013</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=AJAX+crawling%3A+Guide+for+webmasters+and+developers&rft_id=https%3A%2F%2Fsupport.google.com%2Fwebmasters%2Fbin%2Fanswer.py%3Fhl%3Den%26answer%3D174992&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-47"><span class="mw-cite-backlink"><b><a href="#cite_ref-47">^</a></b></span> <span class="reference-text">ITA Labs <a rel="nofollow" class="external text" href="http://semanticweb.com/more-semantics-for-google-ita-software-acquisition-comes-with-needlebase-too_b19329">"ITA Labs Acquisition"</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20140318134631/http://semanticweb.com/more-semantics-for-google-ita-software-acquisition-comes-with-needlebase-too_b19329">Archived</a> 18 March 2014 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a> 20 April 2011 1:28 AM</span> </li> <li id="cite_note-48"><span class="mw-cite-backlink"><b><a href="#cite_ref-48">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://support.apple.com/en-us/HT204683">"About Applebot"</a>. Apple Inc<span class="reference-accessdate">. Retrieved <span class="nowrap">18 October</span> 2021</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=About+Applebot&rft.pub=Apple+Inc&rft_id=https%3A%2F%2Fsupport.apple.com%2Fen-us%2FHT204683&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-Norton-2007-01-25-49"><span class="mw-cite-backlink"><b><a href="#cite_ref-Norton-2007-01-25_49-0">^</a></b></span> <span class="reference-text"> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFNorton2007" class="citation news cs1">Norton, Quinn (25 January 2007). <a rel="nofollow" class="external text" href="https://www.wired.com/2007/01/tax-takers-send-in-the-spiders/">"Tax takers send in the spiders"</a>. Business. <i><a href="/wiki/Wired_(magazine)" title="Wired (magazine)">Wired</a></i>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20161222185208/https://www.wired.com/2007/01/tax-takers-send-in-the-spiders/">Archived</a> from the original on 22 December 2016<span class="reference-accessdate">. Retrieved <span class="nowrap">13 October</span> 2017</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=Wired&rft.atitle=Tax+takers+send+in+the+spiders&rft.date=2007-01-25&rft.aulast=Norton&rft.aufirst=Quinn&rft_id=https%3A%2F%2Fwww.wired.com%2F2007%2F01%2Ftax-takers-send-in-the-spiders%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> <li id="cite_note-Canada-2017-04-11-50"><span class="mw-cite-backlink"><b><a href="#cite_ref-Canada-2017-04-11_50-0">^</a></b></span> <span class="reference-text"> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.canada.ca/en/revenue-agency/services/about-canada-revenue-agency-cra/protecting-your-privacy/privacy-impact-assessment/xenon-web-crawling-initiative-privacy-impact-assessment-summary.html">"Xenon web crawling initiative: privacy impact assessment (PIA) summary"</a>. Ottawa: Government of Canada. 11 April 2017. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20170925203155/https://www.canada.ca/en/revenue-agency/services/about-canada-revenue-agency-cra/protecting-your-privacy/privacy-impact-assessment/xenon-web-crawling-initiative-privacy-impact-assessment-summary.html">Archived</a> from the original on 25 September 2017<span class="reference-accessdate">. Retrieved <span class="nowrap">13 October</span> 2017</span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Xenon+web+crawling+initiative%3A+privacy+impact+assessment+%28PIA%29+summary&rft.place=Ottawa&rft.pub=Government+of+Canada&rft.date=2017-04-11&rft_id=https%3A%2F%2Fwww.canada.ca%2Fen%2Frevenue-agency%2Fservices%2Fabout-canada-revenue-agency-cra%2Fprotecting-your-privacy%2Fprivacy-impact-assessment%2Fxenon-web-crawling-initiative-privacy-impact-assessment-summary.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AWeb+crawler" class="Z3988"></span></span> </li> </ol></div> <div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Web_crawler&action=edit&section=26" title="Edit section: Further reading"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li>Cho, Junghoo, <a rel="nofollow" class="external text" href="http://oak.cs.ucla.edu/~cho/research/crawl.html">"Web Crawling Project"</a>, UCLA Computer Science Department.</li> <li><a rel="nofollow" class="external text" href="http://www.wiley.com/legacy/compbooks/sonnenreich/history.html">A History of Search Engines</a>, from <a href="/wiki/John_Wiley_%26_Sons" class="mw-redirect" title="John Wiley & Sons">Wiley</a></li> <li><a rel="nofollow" class="external text" href="https://github.com/bedirhan/wivet">WIVET</a> is a benchmarking project by <a href="/wiki/OWASP" title="OWASP">OWASP</a>, which aims to measure if a web crawler can identify all the hyperlinks in a target website.</li> <li>Shestakov, Denis, <a rel="nofollow" class="external text" href="https://www.slideshare.net/denshe/icwe13-tutorial-webcrawling">"Current Challenges in Web Crawling"</a> and <a rel="nofollow" class="external text" href="https://www.slideshare.net/denshe/intelligent-crawling-shestakovwiiat13">"Intelligent Web Crawling"</a>, slides for tutorials given at ICWE'13 and WI-IAT'13.</li></ul> <div class="navbox-styles"><style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="Internet_search" style="padding:3px"><table class="nowraplinks mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Internet_search" title="Template:Internet search"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Internet_search" title="Template talk:Internet search"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Internet_search" title="Special:EditPage/Template:Internet search"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Internet_search" style="font-size:114%;margin:0 4em"><a href="/wiki/Internet_search" class="mw-redirect" title="Internet search">Internet search</a></div></th></tr><tr><th scope="row" class="navbox-group" style="width:1%">Types</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Web_search_engine" class="mw-redirect" title="Web search engine">Web search engine</a> (<a href="/wiki/List_of_search_engines" title="List of search engines">List</a>)</li> <li><a href="/wiki/Metasearch_engine" title="Metasearch engine">Metasearch engine</a></li> <li><a href="/wiki/Multimedia_search" title="Multimedia search">Multimedia search</a></li> <li><a href="/wiki/Collaborative_search_engine" title="Collaborative search engine">Collaborative search engine</a></li> <li><a href="/wiki/Cross-language_search" class="mw-redirect" title="Cross-language search">Cross-language search</a></li> <li><a href="/wiki/Local_search_(Internet)" title="Local search (Internet)">Local search</a></li> <li><a href="/wiki/Vertical_search" title="Vertical search">Vertical search</a></li> <li><a href="/wiki/Social_search" title="Social search">Social search</a></li> <li><a href="/wiki/Image_retrieval" title="Image retrieval">Image search</a></li> <li><a href="/wiki/Audio_search_engine" title="Audio search engine">Audio search</a></li> <li><a href="/wiki/Video_search_engine" title="Video search engine">Video search engine</a></li> <li><a href="/wiki/Enterprise_search" title="Enterprise search">Enterprise search</a></li> <li><a href="/wiki/Semantic_search" title="Semantic search">Semantic search</a></li> <li><a href="/wiki/Natural_language_search_engine" class="mw-redirect" title="Natural language search engine">Natural language search engine</a></li> <li><a href="/wiki/Voice_search" title="Voice search">Voice search</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Tools</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Cross-language_information_retrieval" title="Cross-language information retrieval">Cross-language information retrieval</a></li> <li><a href="/wiki/Search_by_sound" title="Search by sound">Search by sound</a></li> <li><a href="/wiki/Search_engine_marketing" title="Search engine marketing">Search engine marketing</a></li> <li><a href="/wiki/Search_engine_optimization" title="Search engine optimization">Search engine optimization</a></li> <li><a href="/wiki/Evaluation_measures_(information_retrieval)" title="Evaluation measures (information retrieval)">Evaluation measures</a></li> <li><a href="/w/index.php?title=Search_oriented_architecture&action=edit&redlink=1" class="new" title="Search oriented architecture (page does not exist)">Search oriented architecture</a></li> <li><a href="/wiki/Selection-based_search" title="Selection-based search">Selection-based search</a></li> <li><a href="/wiki/Document_retrieval" title="Document retrieval">Document retrieval</a></li> <li><a href="/wiki/Text_mining" title="Text mining">Text mining</a></li> <li><a class="mw-selflink selflink">Web crawler</a></li> <li><a href="/wiki/Multisearch" title="Multisearch">Multisearch</a></li> <li><a href="/wiki/Federated_search" title="Federated search">Federated search</a></li> <li><a href="/wiki/Search_aggregator" title="Search aggregator">Search aggregator</a></li> <li><a href="/wiki/Index_(search_engine)" class="mw-redirect" title="Index (search engine)">Index</a>/<a href="/wiki/Web_indexing" title="Web indexing">Web indexing</a></li> <li><a href="/wiki/Focused_crawler" title="Focused crawler">Focused crawler</a></li> <li><a href="/wiki/Spider_trap" title="Spider trap">Spider trap</a></li> <li><a href="/wiki/Robots_exclusion_standard" class="mw-redirect" title="Robots exclusion standard">Robots exclusion standard</a></li> <li><a href="/wiki/Distributed_web_crawling" title="Distributed web crawling">Distributed web crawling</a></li> <li><a href="/wiki/Web_archiving" title="Web archiving">Web archiving</a></li> <li><a href="/wiki/Website_mirroring_software" class="mw-redirect" title="Website mirroring software">Website mirroring software</a></li> <li><a href="/wiki/Web_query" title="Web query">Web query</a></li> <li><a href="/wiki/Web_query_classification" title="Web query classification">Web query classification</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Protocols<br />and standards</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Z39.50" title="Z39.50">Z39.50</a></li> <li><a href="/wiki/Search/Retrieve_Web_Service" title="Search/Retrieve Web Service">Search/Retrieve Web Service</a></li> <li><a href="/wiki/Search/Retrieve_via_URL" title="Search/Retrieve via URL">Search/Retrieve via URL</a></li> <li><a href="/wiki/OpenSearch" title="OpenSearch">OpenSearch</a></li> <li><a href="/wiki/Representational_State_Transfer" class="mw-redirect" title="Representational State Transfer">Representational State Transfer</a></li> <li><a href="/wiki/Wide_area_information_server" title="Wide area information server">Wide area information server</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">See also</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Search_engine_(computing)" title="Search engine (computing)">Search engine</a></li> <li><a href="/wiki/Desktop_search" title="Desktop search">Desktop search</a></li> <li><a href="/wiki/Online_search" title="Online search">Online search</a></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235"></div><div role="navigation" class="navbox" aria-labelledby="Web_crawlers" style="padding:3px"><table class="nowraplinks mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1239400231"><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:Web_crawlers" title="Template:Web crawlers"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:Web_crawlers" title="Template talk:Web crawlers"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:Web_crawlers" title="Special:EditPage/Template:Web crawlers"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="Web_crawlers" style="font-size:114%;margin:0 4em"><a class="mw-selflink selflink">Web crawlers</a></div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div><a href="/wiki/Internet_bot" title="Internet bot">Internet bots</a> designed for <a href="/wiki/Web_crawling" class="mw-redirect" title="Web crawling">Web crawling</a> and <a href="/wiki/Web_indexing" title="Web indexing">Web indexing</a></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Active</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/80legs" title="80legs">80legs</a></li> <li><a href="/wiki/Bingbot" title="Bingbot">bingbot</a></li> <li><a href="/wiki/Crawljax" title="Crawljax">Crawljax</a></li> <li><a href="/wiki/Fetcher" class="mw-redirect" title="Fetcher">Fetcher</a></li> <li><a href="/wiki/Googlebot" title="Googlebot">Googlebot</a></li> <li><a href="/wiki/Heritrix" title="Heritrix">Heritrix</a></li> <li><a href="/wiki/HTTrack" title="HTTrack">HTTrack</a></li> <li><a href="/wiki/PowerMapper" title="PowerMapper">PowerMapper</a></li> <li><a href="/wiki/Wget" title="Wget">Wget</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Discontinued</th><td class="navbox-list-with-group navbox-list navbox-even hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/FAST_Crawler" class="mw-redirect" title="FAST Crawler">FAST Crawler</a></li> <li><a href="/wiki/Msnbot" title="Msnbot">msnbot</a></li> <li><a href="/wiki/RBSE" class="mw-redirect" title="RBSE">RBSE</a></li> <li><a href="/wiki/TkWWW_robot" class="mw-redirect" title="TkWWW robot">TkWWW robot</a></li> <li><a href="/wiki/Twiceler" class="mw-redirect" title="Twiceler">Twiceler</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Types</th><td class="navbox-list-with-group navbox-list navbox-odd hlist" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Distributed_web_crawler" class="mw-redirect" title="Distributed web crawler">Distributed web crawler</a></li> <li><a href="/wiki/Focused_crawler" title="Focused crawler">Focused crawler</a></li></ul> </div></td></tr></tbody></table></div> <div class="navbox-styles"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1236075235"></div><div role="navigation" class="navbox authority-control" aria-label="Navbox" style="padding:3px"><table class="nowraplinks hlist navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Help:Authority_control" title="Help:Authority control">Authority control databases</a>: National <span class="mw-valign-text-top noprint" typeof="mw:File/Frameless"><a href="https://www.wikidata.org/wiki/Q45842#identifiers" title="Edit this at Wikidata"><img alt="Edit this at Wikidata" src="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/10px-OOjs_UI_icon_edit-ltr-progressive.svg.png" decoding="async" width="10" height="10" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/15px-OOjs_UI_icon_edit-ltr-progressive.svg.png 1.5x, //upload.wikimedia.org/wikipedia/en/thumb/8/8a/OOjs_UI_icon_edit-ltr-progressive.svg/20px-OOjs_UI_icon_edit-ltr-progressive.svg.png 2x" data-file-width="20" data-file-height="20" /></a></span></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"><ul><li><span class="uid"><a rel="nofollow" class="external text" href="https://d-nb.info/gnd/4796298-7">Germany</a></span></li></ul></div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Web_crawler&oldid=1257967634">https://en.wikipedia.org/w/index.php?title=Web_crawler&oldid=1257967634</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Search_engine_software" title="Category:Search engine software">Search engine software</a></li><li><a href="/wiki/Category:Web_crawlers" title="Category:Web crawlers">Web crawlers</a></li><li><a href="/wiki/Category:Internet_search_algorithms" title="Category:Internet search algorithms">Internet search algorithms</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:Webarchive_template_wayback_links" title="Category:Webarchive template wayback links">Webarchive template wayback links</a></li><li><a href="/wiki/Category:CS1_maint:_multiple_names:_authors_list" title="Category:CS1 maint: multiple names: authors list">CS1 maint: multiple names: authors list</a></li><li><a href="/wiki/Category:CS1_errors:_missing_periodical" title="Category:CS1 errors: missing periodical">CS1 errors: missing periodical</a></li><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_is_different_from_Wikidata" title="Category:Short description is different from Wikidata">Short description is different from Wikidata</a></li><li><a href="/wiki/Category:Use_dmy_dates_from_September_2020" title="Category:Use dmy dates from September 2020">Use dmy dates from September 2020</a></li><li><a href="/wiki/Category:All_articles_with_unsourced_statements" title="Category:All articles with unsourced statements">All articles with unsourced statements</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_February_2023" title="Category:Articles with unsourced statements from February 2023">Articles with unsourced statements from February 2023</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_June_2021" title="Category:Articles with unsourced statements from June 2021">Articles with unsourced statements from June 2021</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 17 November 2024, at 12:37<span class="anonymous-show"> (UTC)</span>.</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Web_crawler&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-sf2cc","wgBackendResponseTime":186,"wgPageParseReport":{"limitreport":{"cputime":"0.722","walltime":"0.970","ppvisitednodes":{"value":3764,"limit":1000000},"postexpandincludesize":{"value":110287,"limit":2097152},"templateargumentsize":{"value":2625,"limit":2097152},"expansiondepth":{"value":12,"limit":100},"expensivefunctioncount":{"value":14,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":164849,"limit":5000000},"entityaccesscount":{"value":1,"limit":400},"timingprofile":["100.00% 799.066 1 -total"," 45.48% 363.381 1 Template:Reflist"," 13.63% 108.903 5 Template:Cite_web"," 13.19% 105.370 1 Template:Short_description"," 11.34% 90.618 1 Template:Internet_search"," 11.25% 89.887 2 Template:Navbox"," 11.23% 89.752 16 Template:Cite_journal"," 9.02% 72.105 7 Template:Cite_book"," 7.59% 60.675 1 Template:Hatnote_group"," 6.59% 52.671 7 Template:Main_other"]},"scribunto":{"limitreport-timeusage":{"value":"0.431","limit":"10.000"},"limitreport-memusage":{"value":7150597,"limit":52428800}},"cachereport":{"origin":"mw-web.codfw.main-f69cdc8f6-4qzqr","timestamp":"20241122140519","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Web crawler","url":"https:\/\/en.wikipedia.org\/wiki\/Web_crawler","sameAs":"http:\/\/www.wikidata.org\/entity\/Q45842","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q45842","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2001-04-27T04:55:28Z","dateModified":"2024-11-17T12:37:46Z","image":"https:\/\/upload.wikimedia.org\/wikipedia\/commons\/d\/df\/WebCrawlerArchitecture.svg","headline":"internet bot that systematically browses the World Wide Web, typically for the purpose of Web indexing (web spidering)"}</script> </body> </html>