CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Data deduplication - Wikipedia</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )enwikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"7a247205-6162-4607-8679-60d1d79006c6","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Data_deduplication","wgTitle":"Data deduplication","wgCurRevisionId":1232070507,"wgRevisionId":1232070507,"wgArticleId":17174890,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Webarchive template wayback links","Articles with short description","Short description matches Wikidata","All articles with unsourced statements","Articles with unsourced statements from August 2016","Articles with unsourced statements from December 2023","Data compression","Data management"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Data_deduplication","wgRelevantArticleId":17174890,"wgIsProbablyEditable" :true,"wgRelevantPageIsProbablyEditable":true,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikipedia","wgCiteReferencePreviewsActive":false,"wgFlaggedRevsParams":{"tags":{"status":{"levels":1}}},"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgPopupsFlags":0,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":20000,"wgRelatedArticlesCompat":[],"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgULSPosition":"interlanguage","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgWikibaseItemId":"Q1182260","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"], "GEHomepageSuggestedEditsEnableTopics":true,"wgGETopicsMatchModeEnabled":false,"wgGEStructuredTaskRejectionReasonTextInputEnabled":false,"wgGELevelingUpEnabledForUser":false};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.cite.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","jquery.makeCollapsible.styles":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","wikibase.client.init":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["ext.cite.ux-enhancements","site","mediawiki.page.ready","jquery.makeCollapsible","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.ReferenceTooltips","ext.gadget.switcher","ext.urlShortener.toolbar","ext.centralauth.centralautologin", "ext.popups","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.interface","ext.cx.eventlogging.campaigns","ext.cx.uls.quick.actions","wikibase.client.vector-2022","ext.checkUser.clientHints","ext.growthExperiments.SuggestedEditSession","wikibase.sidebar.tracking"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.cite.styles%7Cext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cext.wikimediamessages.styles%7Cjquery.makeCollapsible.styles%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles%7Cwikibase.client.init&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Data deduplication - Wikipedia"> <meta property="og:type" content="website"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//en.m.wikipedia.org/wiki/Data_deduplication"> <link rel="alternate" type="application/x-wiki" title="Edit this page" href="/w/index.php?title=Data_deduplication&action=edit"> <link rel="apple-touch-icon" href="/static/apple-touch/wikipedia.png"> <link rel="icon" href="/static/favicon/wikipedia.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikipedia (en)"> <link rel="EditURI" type="application/rsd+xml" href="//en.wikipedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://en.wikipedia.org/wiki/Data_deduplication"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en"> <link rel="alternate" type="application/atom+xml" title="Wikipedia Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-Data_deduplication rootpage-Data_deduplication skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-contents" class="mw-list-item"><a href="/wiki/Wikipedia:Contents" title="Guides to browsing Wikipedia"><span>Contents</span></a></li><li id="n-currentevents" class="mw-list-item"><a href="/wiki/Portal:Current_events" title="Articles related to current events"><span>Current events</span></a></li><li id="n-randompage" class="mw-list-item"><a href="/wiki/Special:Random" title="Visit a randomly selected article [x]" accesskey="x"><span>Random article</span></a></li><li id="n-aboutsite" class="mw-list-item"><a href="/wiki/Wikipedia:About" title="Learn about Wikipedia and how it works"><span>About Wikipedia</span></a></li><li id="n-contactpage" class="mw-list-item"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us" title="How to contact Wikipedia"><span>Contact us</span></a></li> </ul> </div> </div> <div id="p-interaction" class="vector-menu mw-portlet mw-portlet-interaction" > <div class="vector-menu-heading"> Contribute </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-help" class="mw-list-item"><a href="/wiki/Help:Contents" title="Guidance on how to use and edit Wikipedia"><span>Help</span></a></li><li id="n-introduction" class="mw-list-item"><a href="/wiki/Help:Introduction" title="Learn how to edit Wikipedia"><span>Learn to edit</span></a></li><li id="n-portal" class="mw-list-item"><a href="/wiki/Wikipedia:Community_portal" title="The hub for editors"><span>Community portal</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes to Wikipedia [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_upload_wizard" title="Add images or other media for use on Wikipedia"><span>Upload file</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikipedia.png" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikipedia" src="/static/images/mobile/copyright/wikipedia-wordmark-en.svg" style="width: 7.5em; height: 1.125em;"> <img class="mw-logo-tagline" alt="The Free Encyclopedia" src="/static/images/mobile/copyright/wikipedia-tagline-en.svg" width="117" height="13" style="width: 7.3125em; height: 0.8125em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikipedia [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikipedia" aria-label="Search Wikipedia" autocapitalize="sentences" title="Search Wikipedia [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-createaccount-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:CreateAccount&returnto=Data+deduplication" title="You are encouraged to create an account and log in; however, it is not mandatory" class=""><span>Create account</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Data+deduplication" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out" title="Log in and more options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-createaccount" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:CreateAccount&returnto=Data+deduplication" title="You are encouraged to create an account and log in; however, it is not mandatory"><span class="vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd"></span> <span>Create account</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Data+deduplication" title="You're encouraged to log in; however, it's not mandatory. [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> <div id="p-user-menu-anon-editor" class="vector-menu mw-portlet mw-portlet-user-menu-anon-editor" > <div class="vector-menu-heading"> Pages for logged out editors <a href="/wiki/Help:Introduction" aria-label="Learn more about editing"><span>learn more</span></a> </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-anoncontribs" class="mw-list-item"><a href="/wiki/Special:MyContributions" title="A list of edits made from this IP address [y]" accesskey="y"><span>Contributions</span></a></li><li id="pt-anontalk" class="mw-list-item"><a href="/wiki/Special:MyTalk" title="Discussion about edits from this IP address [n]" accesskey="n"><span>Talk</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">(Top)</div> </a> </li> <li id="toc-Functioning_principle" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Functioning_principle"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Functioning principle</span> </div> </a> <ul id="toc-Functioning_principle-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Benefits" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Benefits"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Benefits</span> </div> </a> <ul id="toc-Benefits-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Classification" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Classification"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Classification</span> </div> </a> <button aria-controls="toc-Classification-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Classification subsection</span> </button> <ul id="toc-Classification-sublist" class="vector-toc-list"> <li id="toc-Post-process_versus_in-line_deduplication" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Post-process_versus_in-line_deduplication"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Post-process versus in-line deduplication</span> </div> </a> <ul id="toc-Post-process_versus_in-line_deduplication-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Data_formats" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Data_formats"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.2</span> <span>Data formats</span> </div> </a> <ul id="toc-Data_formats-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Source_versus_target_deduplication" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Source_versus_target_deduplication"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.3</span> <span>Source versus target deduplication</span> </div> </a> <ul id="toc-Source_versus_target_deduplication-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deduplication_methods" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Deduplication_methods"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.4</span> <span>Deduplication methods</span> </div> </a> <ul id="toc-Deduplication_methods-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Single_instance_storage" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Single_instance_storage"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Single instance storage</span> </div> </a> <ul id="toc-Single_instance_storage-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Drawbacks_and_concerns" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Drawbacks_and_concerns"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Drawbacks and concerns</span> </div> </a> <ul id="toc-Drawbacks_and_concerns-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Implementations" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Implementations"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>Implementations</span> </div> </a> <ul id="toc-Implementations-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-See_also" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#See_also"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>See also</span> </div> </a> <ul id="toc-See_also-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-External_links" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#External_links"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>External links</span> </div> </a> <ul id="toc-External_links-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Data deduplication</span></h1> <div id="p-lang-btn" class="vector-dropdown mw-portlet mw-portlet-lang" > <input type="checkbox" id="p-lang-btn-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-p-lang-btn" class="vector-dropdown-checkbox mw-interlanguage-selector" aria-label="Go to an article in another language. Available in 14 languages" > <label id="p-lang-btn-label" for="p-lang-btn-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive mw-portlet-lang-heading-14" aria-hidden="true" ><span class="vector-icon mw-ui-icon-language-progressive mw-ui-icon-wikimedia-language-progressive"></span> <span class="vector-dropdown-label-text">14 languages</span> </label> <div class="vector-dropdown-content"> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li class="interlanguage-link interwiki-ar mw-list-item"><a href="https://ar.wikipedia.org/wiki/%D8%A5%D8%B2%D8%A7%D9%84%D8%A9_%D8%A7%D9%84%D8%A8%D9%8A%D8%A7%D9%86%D8%A7%D8%AA_%D8%A7%D9%84%D9%85%D9%83%D8%B1%D8%B1%D8%A9" title="إزالة البيانات المكررة – Arabic" lang="ar" hreflang="ar" data-title="إزالة البيانات المكررة" data-language-autonym="العربية" data-language-local-name="Arabic" class="interlanguage-link-target"><span>العربية</span></a></li><li class="interlanguage-link interwiki-cs mw-list-item"><a href="https://cs.wikipedia.org/wiki/Deduplikace" title="Deduplikace – Czech" lang="cs" hreflang="cs" data-title="Deduplikace" data-language-autonym="Čeština" data-language-local-name="Czech" class="interlanguage-link-target"><span>Čeština</span></a></li><li class="interlanguage-link interwiki-de mw-list-item"><a href="https://de.wikipedia.org/wiki/Deduplikation" title="Deduplikation – German" lang="de" hreflang="de" data-title="Deduplikation" data-language-autonym="Deutsch" data-language-local-name="German" class="interlanguage-link-target"><span>Deutsch</span></a></li><li class="interlanguage-link interwiki-es mw-list-item"><a href="https://es.wikipedia.org/wiki/Deduplicaci%C3%B3n_de_datos" title="Deduplicación de datos – Spanish" lang="es" hreflang="es" data-title="Deduplicación de datos" data-language-autonym="Español" data-language-local-name="Spanish" class="interlanguage-link-target"><span>Español</span></a></li><li class="interlanguage-link interwiki-fr mw-list-item"><a href="https://fr.wikipedia.org/wiki/D%C3%A9duplication" title="Déduplication – French" lang="fr" hreflang="fr" data-title="Déduplication" data-language-autonym="Français" data-language-local-name="French" class="interlanguage-link-target"><span>Français</span></a></li><li class="interlanguage-link interwiki-hr mw-list-item"><a href="https://hr.wikipedia.org/wiki/Deduplikacija" title="Deduplikacija – Croatian" lang="hr" hreflang="hr" data-title="Deduplikacija" data-language-autonym="Hrvatski" data-language-local-name="Croatian" class="interlanguage-link-target"><span>Hrvatski</span></a></li><li class="interlanguage-link interwiki-id mw-list-item"><a href="https://id.wikipedia.org/wiki/Deduplikasi_data" title="Deduplikasi data – Indonesian" lang="id" hreflang="id" data-title="Deduplikasi data" data-language-autonym="Bahasa Indonesia" data-language-local-name="Indonesian" class="interlanguage-link-target"><span>Bahasa Indonesia</span></a></li><li class="interlanguage-link interwiki-kk mw-list-item"><a href="https://kk.wikipedia.org/wiki/%D0%9C%D3%99%D0%BB%D1%96%D0%BC%D0%B5%D1%82%D1%82%D0%B5%D1%80%D0%B4%D1%96_%D2%9B%D0%B0%D0%B9%D1%82%D0%B0%D0%BB%D0%B0%D1%83" title="Мәліметтерді қайталау – Kazakh" lang="kk" hreflang="kk" data-title="Мәліметтерді қайталау" data-language-autonym="Қазақша" data-language-local-name="Kazakh" class="interlanguage-link-target"><span>Қазақша</span></a></li><li class="interlanguage-link interwiki-no mw-list-item"><a href="https://no.wikipedia.org/wiki/Datadeduplisering" title="Datadeduplisering – Norwegian Bokmål" lang="nb" hreflang="nb" data-title="Datadeduplisering" data-language-autonym="Norsk bokmål" data-language-local-name="Norwegian Bokmål" class="interlanguage-link-target"><span>Norsk bokmål</span></a></li><li class="interlanguage-link interwiki-pl mw-list-item"><a href="https://pl.wikipedia.org/wiki/Deduplikacja" title="Deduplikacja – Polish" lang="pl" hreflang="pl" data-title="Deduplikacja" data-language-autonym="Polski" data-language-local-name="Polish" class="interlanguage-link-target"><span>Polski</span></a></li><li class="interlanguage-link interwiki-pt mw-list-item"><a href="https://pt.wikipedia.org/wiki/Desduplica%C3%A7%C3%A3o_de_dados" title="Desduplicação de dados – Portuguese" lang="pt" hreflang="pt" data-title="Desduplicação de dados" data-language-autonym="Português" data-language-local-name="Portuguese" class="interlanguage-link-target"><span>Português</span></a></li><li class="interlanguage-link interwiki-ru mw-list-item"><a href="https://ru.wikipedia.org/wiki/%D0%94%D0%B5%D0%B4%D1%83%D0%BF%D0%BB%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F" title="Дедупликация – Russian" lang="ru" hreflang="ru" data-title="Дедупликация" data-language-autonym="Русский" data-language-local-name="Russian" class="interlanguage-link-target"><span>Русский</span></a></li><li class="interlanguage-link interwiki-uk mw-list-item"><a href="https://uk.wikipedia.org/wiki/%D0%94%D0%B5%D0%B4%D1%83%D0%B1%D0%BB%D1%96%D0%BA%D0%B0%D1%86%D1%96%D1%8F" title="Дедублікація – Ukrainian" lang="uk" hreflang="uk" data-title="Дедублікація" data-language-autonym="Українська" data-language-local-name="Ukrainian" class="interlanguage-link-target"><span>Українська</span></a></li><li class="interlanguage-link interwiki-zh mw-list-item"><a href="https://zh.wikipedia.org/wiki/%E9%87%8D%E5%A4%8D%E6%95%B0%E6%8D%AE%E5%88%A0%E9%99%A4" title="重复数据删除 – Chinese" lang="zh" hreflang="zh" data-title="重复数据删除" data-language-autonym="中文" data-language-local-name="Chinese" class="interlanguage-link-target"><span>中文</span></a></li> </ul> <div class="after-portlet after-portlet-lang"><span class="wb-langlinks-edit wb-langlinks-link"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q1182260#sitelinks-wikipedia" title="Edit interlanguage links" class="wbc-editpage">Edit links</a></span></div> </div> </div> </div> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Data_deduplication" title="View the content page [c]" accesskey="c"><span>Article</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Data_deduplication" rel="discussion" title="Discuss improvements to the content page [t]" accesskey="t"><span>Talk</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Data_deduplication"><span>Read</span></a></li><li id="ca-edit" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Data_deduplication&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Data_deduplication&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Data_deduplication"><span>Read</span></a></li><li id="ca-more-edit" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Data_deduplication&action=edit" title="Edit this page [e]" accesskey="e"><span>Edit</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Data_deduplication&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Data_deduplication" title="List of all English Wikipedia pages containing links to this page [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Data_deduplication" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-upload" class="mw-list-item"><a href="/wiki/Wikipedia:File_Upload_Wizard" title="Upload files [u]" accesskey="u"><span>Upload file</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Data_deduplication&oldid=1232070507" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Data_deduplication&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Data_deduplication&id=1232070507&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FData_deduplication"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FData_deduplication"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Data_deduplication&action=show-download-screen" title="Download this page as a PDF file"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Data_deduplication&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> <div id="p-wikibase-otherprojects" class="vector-menu mw-portlet mw-portlet-wikibase-otherprojects" > <div class="vector-menu-heading"> In other projects </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-wikibase" class="wb-otherproject-link wb-otherproject-wikibase-dataitem mw-list-item"><a href="https://www.wikidata.org/wiki/Special:EntityPage/Q1182260" title="Structured data on this page hosted by Wikidata [g]" accesskey="g"><span>Wikidata item</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikipedia, the free encyclopedia</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><div class="shortdescription nomobile noexcerpt noprint searchaux" style="display:none">Data processing technique to eliminate duplicate copies of repeating data</div> <p>In <a href="/wiki/Computing" title="Computing">computing</a>, <b>data deduplication</b> is a technique for eliminating duplicate copies of repeating data. Successful implementation of the technique can improve storage utilization, which may in turn lower <a href="/wiki/Capital_expenditure" title="Capital expenditure">capital expenditure</a> by reducing the overall amount of storage media required to meet storage capacity needs. It can also be applied to network data transfers to reduce the number of bytes that must be sent. </p><p>The deduplication process requires comparison of data 'chunks' (also known as 'byte patterns') which are unique, contiguous blocks of data. These chunks are identified and stored during a process of analysis, and compared to other chunks within existing data. Whenever a match occurs, the redundant chunk is replaced with a small reference that points to the stored chunk. Given that the same byte pattern may occur dozens, hundreds, or even thousands of times (the match frequency is dependent on the chunk size), the amount of data that must be stored or transferred can be greatly reduced.<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-snia_2-0" class="reference"><a href="#cite_note-snia-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p><p>A related technique is <a href="/wiki/Single-instance_storage" title="Single-instance storage">single-instance (data) storage</a>, which replaces multiple copies of content at the whole-file level with a single shared copy. While possible to combine this with other forms of data compression and deduplication, it is distinct from newer approaches to data deduplication (which can operate at the segment or sub-block level). </p><p>Deduplication is different from data compression algorithms, such as <a href="/wiki/LZ77_and_LZ78" title="LZ77 and LZ78">LZ77 and LZ78</a>. Whereas compression algorithms identify redundant data inside individual files and encodes this redundant data more efficiently, the intent of deduplication is to inspect large volumes of data and identify large sections – such as entire files or large sections of files – that are identical, and replace them with a shared copy. </p> <meta property="mw:PageProp/toc" /> <div class="mw-heading mw-heading2"><h2 id="Functioning_principle">Functioning principle</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=1" title="Edit section: Functioning principle"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>For example, a typical email system might contain 100 instances of the same 1 MB (<a href="/wiki/Megabyte" title="Megabyte">megabyte</a>) file attachment. Each time the <a href="/wiki/Email" title="Email">email</a> platform is backed up, all 100 instances of the attachment are saved, requiring 100 MB storage space. With data deduplication, only one instance of the attachment is actually stored; the subsequent instances are referenced back to the saved copy for deduplication ratio of roughly 100 to 1. Deduplication is often paired with data compression for additional storage saving: Deduplication is first used to eliminate large chunks of repetitive data, and compression is then used to efficiently encode each of the stored chunks.<sup id="cite_ref-3" class="reference"><a href="#cite_note-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup> </p><p>In <a href="/wiki/Source_code" title="Source code">computer code</a>, deduplication is done by, for example, storing information in <a href="/wiki/Variable_(computer_science)" title="Variable (computer science)">variables</a> so that they don't have to be written out individually but can be changed all at once at a central <a href="/wiki/Reference_(computer_science)" title="Reference (computer science)">referenced</a> location. Examples are <a href="/wiki/CSS_class" class="mw-redirect" title="CSS class">CSS classes</a> and <a href="/wiki/Help:Footnotes#Footnotes:_using_a_source_more_than_once" title="Help:Footnotes">named references</a> in <a href="/wiki/MediaWiki" title="MediaWiki">MediaWiki</a>. </p> <div class="mw-heading mw-heading2"><h2 id="Benefits">Benefits</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=2" title="Edit section: Benefits"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Storage-based data deduplication reduces the amount of storage needed for a given set of files. It is most effective in applications where many copies of very similar or even identical data are stored on a single disk. In the case of data backups, which routinely are performed to protect against data loss, most data in a given backup remain unchanged from the previous backup. Common backup systems try to exploit this by omitting (or <a href="/wiki/Hard_link" title="Hard link">hard linking</a>) files that haven't changed or storing <a href="/wiki/Data_differencing" title="Data differencing">differences</a> between files. Neither approach captures all redundancies, however. Hard-linking does not help with large files that have only changed in small ways, such as an email database; differences only find redundancies in adjacent versions of a single file (consider a section that was deleted and later added in again, or a logo image included in many documents). </p><p>In-line network data deduplication is used to reduce the number of bytes that must be transferred between endpoints, which can reduce the amount of bandwidth required. See <a href="/wiki/WAN_optimization" title="WAN optimization">WAN optimization</a> for more information. </p><p>Virtual servers and virtual desktops benefit from deduplication because it allows nominally separate system files for each virtual machine to be coalesced into a single storage space. At the same time, if a given virtual machine customizes a file, deduplication will not change the files on the other virtual machines—something that alternatives like hard links or shared disks do not offer. Backing up or making duplicate copies of virtual environments is similarly improved. </p> <div class="mw-heading mw-heading2"><h2 id="Classification">Classification</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=3" title="Edit section: Classification"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <div class="mw-heading mw-heading3"><h3 id="Post-process_versus_in-line_deduplication">Post-process versus in-line deduplication</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=4" title="Edit section: Post-process versus in-line deduplication"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Deduplication may occur "in-line", as data is flowing, or "post-process" after it has been written. </p><p>With post-process deduplication, new data is first stored on the storage device and then a process at a later time will analyze the data looking for duplication. The benefit is that there is no need to wait for the <a href="/wiki/Hash_function" title="Hash function">hash</a> calculations and lookup to be completed before storing the data, thereby ensuring that store performance is not degraded. Implementations offering policy-based operation can give users the ability to defer optimization on "active" files, or to process files based on type and location. One potential drawback is that duplicate data may be unnecessarily stored for a short time, which can be problematic if the system is nearing full capacity. </p><p>Alternatively, deduplication hash calculations can be done in-line: synchronized as data enters the target device. If the storage system identifies a block which it has already stored, only a reference to the existing block is stored, rather than the whole new block. </p><p>The advantage of in-line deduplication over post-process deduplication is that it requires less storage and network traffic, since duplicate data is never stored or transferred. On the negative side, hash calculations may be computationally expensive, thereby reducing the storage throughput. However, certain vendors with in-line deduplication have demonstrated equipment which performs in-line deduplication at high rates. </p><p>Post-process and in-line deduplication methods are often heavily debated.<sup id="cite_ref-4" class="reference"><a href="#cite_note-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-5" class="reference"><a href="#cite_note-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading3"><h3 id="Data_formats">Data formats</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=5" title="Edit section: Data formats"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>The <a href="/wiki/Storage_Networking_Industry_Association" title="Storage Networking Industry Association">SNIA Dictionary</a> identifies two methods:<sup id="cite_ref-snia_2-1" class="reference"><a href="#cite_note-snia-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> </p> <ul><li>content-agnostic data deduplication - a data deduplication method that does not require awareness of specific application data formats.</li> <li>content-aware data deduplication - a data deduplication method that leverages knowledge of specific application data formats.</li></ul> <div class="mw-heading mw-heading3"><h3 id="Source_versus_target_deduplication">Source versus target deduplication</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=6" title="Edit section: Source versus target deduplication"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Another way to classify data deduplication methods is according to where they occur. Deduplication occurring close to where data is created, is referred to as "source deduplication". When it occurs near where the data is stored, it is called "target deduplication". </p><p>Source deduplication ensures that data on the data source is deduplicated. This generally takes place directly within a file system. The file system will periodically scan new files creating hashes and compare them to hashes of existing files. When files with same hashes are found then the file copy is removed and the new file points to the old file. Unlike <a href="/wiki/Hard_link" title="Hard link">hard links</a> however, duplicated files are considered to be separate entities and if one of the duplicated files is later modified, then using a system called <a href="/wiki/Copy-on-write" title="Copy-on-write">copy-on-write</a> a copy of that changed file or block is created. The deduplication process is transparent to the users and backup applications. Backing up a deduplicated file system will often cause duplication to occur resulting in the backups being bigger than the source data.<sup id="cite_ref-6" class="reference"><a href="#cite_note-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup> </p><p><span class="anchor" id="reflink"></span>Source deduplication can be declared explicitly for copying operations, as no calculation is needed to know that the copied data is in need of deduplication. This leads to a new form of "linking" on file systems called the <b>reflink</b> (Linux) or <b>clonefile</b> (MacOS), where one or more <a href="/wiki/Inode" title="Inode">inodes</a> (file information entries) are made to share some or all of their data. It is named analogously to <a href="/wiki/Hard_link" title="Hard link">hard links</a>, which work at the inode level, and <a href="/wiki/Symbolic_link" title="Symbolic link">symbolic links</a> that work at the filename level.<sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> The individual entries have a copy-on-write behavior that is non-aliasing, i.e. changing one copy afterwards will not affect other copies.<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> Microsoft's <a href="/wiki/ReFS" title="ReFS">ReFS</a> also supports this operation.<sup id="cite_ref-10" class="reference"><a href="#cite_note-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup> </p><p>Target deduplication is the process of removing duplicates when the data was not generated at that location. Example of this would be a server connected to a SAN/NAS, The SAN/NAS would be a target for the server (target deduplication). The server is not aware of any deduplication, the server is also the point of data generation. A second example would be backup. Generally this will be a backup store such as a data repository or a <a href="/wiki/Virtual_tape_library" title="Virtual tape library">virtual tape library</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Deduplication_methods">Deduplication methods</h3><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=7" title="Edit section: Deduplication methods"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>One of the most common forms of data deduplication implementations works by comparing chunks of data to detect duplicates. For that to happen, each chunk of data is assigned an identification, calculated by the software, typically using cryptographic hash functions. In many implementations, the assumption is made that if the identification is identical, the data is identical, even though this cannot be true in all cases due to the <a href="/wiki/Pigeonhole_principle" title="Pigeonhole principle">pigeonhole principle</a>; other implementations do not assume that two blocks of data with the same identifier are identical, but actually verify that data with the same identification is identical.<sup id="cite_ref-11" class="reference"><a href="#cite_note-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup> If the software either assumes that a given identification already exists in the deduplication namespace or actually verifies the identity of the two blocks of data, depending on the implementation, then it will replace that duplicate chunk with a link. </p><p>Once the data has been deduplicated, upon read back of the file, wherever a link is found, the system simply replaces that link with the referenced data chunk. The deduplication process is intended to be transparent to end users and applications. </p><p>Commercial deduplication implementations differ by their chunking methods and architectures. </p> <ul><li>Chunking: In some systems, chunks are defined by physical layer constraints (e.g. 4KB block size in <a href="/wiki/Write_Anywhere_File_Layout" title="Write Anywhere File Layout">WAFL</a>). In some systems only complete files are compared, which is called <a href="/wiki/Single-instance_storage" title="Single-instance storage">single-instance storage</a> or SIS. The most intelligent (but CPU intensive) method to chunking is generally considered to be sliding-block, also called <a href="/wiki/Content-Defined_Chunking" class="mw-redirect" title="Content-Defined Chunking">Content-Defined Chunking</a>. In sliding block, a window is passed along the file stream to seek out more naturally occurring internal file boundaries.</li> <li>Client backup deduplication: This is the process where the deduplication hash calculations are initially created on the source (client) machines. Files that have identical hashes to files already in the target device are not sent, the target device just creates appropriate internal links to reference the duplicated data. The benefit of this is that it avoids data being unnecessarily sent across the network thereby reducing traffic load.</li> <li>Primary storage and secondary storage: By definition, primary storage systems are designed for optimal performance, rather than lowest possible cost. The design criteria for these systems is to increase performance, at the expense of other considerations. Moreover, primary storage systems are much less tolerant of any operation that can negatively impact performance. Also by definition, secondary storage systems contain primarily duplicate, or secondary copies of data. These copies of data are typically not used for actual production operations and as a result are more tolerant of some performance degradation, in exchange for increased efficiency.</li></ul> <p>To date, data deduplication has predominantly been used with secondary storage systems. The reasons for this are two-fold: First, data deduplication requires overhead to discover and remove the duplicate data. In primary storage systems, this overhead may impact performance. The second reason why deduplication is applied to secondary data, is that secondary data tends to have more duplicate data. Backup application in particular commonly generate significant portions of duplicate data over time. </p><p>Data deduplication has been deployed successfully with primary storage in some cases where the system design does not require significant overhead, or impact performance. </p> <div class="mw-heading mw-heading2"><h2 id="Single_instance_storage">Single instance storage</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=8" title="Edit section: Single instance storage"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p><a href="/wiki/Single-instance_storage" title="Single-instance storage">Single-instance storage</a> (SIS) is a system's ability to take multiple copies of content objects and replace them by a single shared copy. It is a means to eliminate data duplication and to increase efficiency. SIS is frequently implemented in <a href="/wiki/File_system" title="File system">file systems</a>, <a href="/wiki/Email_server" class="mw-redirect" title="Email server">email server</a> software, <a href="/wiki/Data" title="Data">data</a> <a href="/wiki/Backup" title="Backup">backup</a>, and other storage-related computer software. Single-instance storage is a simple variant of data deduplication. While data deduplication may work at a segment or sub-block level, single instance storage works at the object level, eliminating redundant copies of objects such as entire files or email messages.<sup id="cite_ref-12" class="reference"><a href="#cite_note-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> </p><p>Single-instance storage can be used alongside (or layered upon) other data duplication or data compression methods to improve performance in exchange for an increase in complexity and for (in some cases) a minor increase in storage space requirements. </p> <div class="mw-heading mw-heading2"><h2 id="Drawbacks_and_concerns">Drawbacks and concerns</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=9" title="Edit section: Drawbacks and concerns"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>One method for deduplicating data relies on the use of <a href="/wiki/Cryptographic_hash_function" title="Cryptographic hash function">cryptographic hash functions</a> to identify duplicate segments of data. If two different pieces of information generate the same hash value, this is known as a <a href="/wiki/Hash_collision" title="Hash collision">collision</a>. The probability of a collision depends mainly on the hash length (see <a href="/wiki/Birthday_attack" title="Birthday attack">birthday attack</a>). Thus, the concern arises that <a href="/wiki/Data_corruption" title="Data corruption">data corruption</a> can occur if a <a href="/wiki/Hash_collision" title="Hash collision">hash collision</a> occurs, and additional means of verification are not used to verify whether there is a difference in data, or not. Both in-line and post-process architectures may offer bit-for-bit validation of original data for guaranteed data integrity. The hash functions used include standards such as <a href="/wiki/SHA-1" title="SHA-1">SHA-1</a>, <a href="/wiki/SHA-256" class="mw-redirect" title="SHA-256">SHA-256</a>, and others. </p><p>The computational resource intensity of the process can be a drawback of data deduplication. To improve performance, some systems utilize both weak and strong hashes. Weak hashes are much faster to calculate but there is a greater risk of a hash collision. Systems that utilize weak hashes will subsequently calculate a strong hash and will use it as the determining factor to whether it is actually the same data or not. Note that the system overhead associated with calculating and looking up hash values is primarily a function of the deduplication workflow. The reconstitution of files does not require this processing and any incremental performance penalty associated with re-assembly of data chunks is unlikely to impact application performance. </p><p>Another concern is the interaction of compression and encryption. The goal of encryption is to eliminate any discernible patterns in the data. Thus encrypted data cannot be deduplicated, even though the underlying data may be redundant. </p><p>Although not a shortcoming of data deduplication, there have been data breaches when insufficient security and access validation procedures are used with large repositories of deduplicated data. In some systems, as typical with cloud storage,<sup class="noprint Inline-Template Template-Fact" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed"><span title="This claim needs references to reliable sources. (August 2016)">citation needed</span></a></i>]</sup> an attacker can retrieve data owned by others by knowing or guessing the hash value of the desired data.<sup id="cite_ref-13" class="reference"><a href="#cite_note-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup> </p> <div class="mw-heading mw-heading2"><h2 id="Implementations">Implementations</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=10" title="Edit section: Implementations"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <p>Deduplication is implemented in some filesystems such as in <a href="/wiki/ZFS" title="ZFS">ZFS</a> or <a href="/wiki/Write_Anywhere_File_Layout" title="Write Anywhere File Layout">Write Anywhere File Layout</a> and in different <a href="/wiki/Disk_arrays" class="mw-redirect" title="Disk arrays">disk arrays</a> models.<sup class="noprint Inline-Template Template-Fact" style="white-space:nowrap;">[<i><a href="/wiki/Wikipedia:Citation_needed" title="Wikipedia:Citation needed"><span title="using blog as a reference (December 2023)">citation needed</span></a></i>]</sup> It is a service available on both <a href="/wiki/NTFS" title="NTFS">NTFS</a> and <a href="/wiki/ReFS" title="ReFS">ReFS</a> on Windows servers. </p> <div class="mw-heading mw-heading2"><h2 id="See_also">See also</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=11" title="Edit section: See also"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1184024115">.mw-parser-output .div-col{margin-top:0.3em;column-width:30em}.mw-parser-output .div-col-small{font-size:90%}.mw-parser-output .div-col-rules{column-rule:1px solid #aaa}.mw-parser-output .div-col dl,.mw-parser-output .div-col ol,.mw-parser-output .div-col ul{margin-top:0}.mw-parser-output .div-col li,.mw-parser-output .div-col dd{page-break-inside:avoid;break-inside:avoid-column}</style><div class="div-col" style="column-width: 20em;"> <ul><li><a href="/wiki/Capacity_optimization" title="Capacity optimization">Capacity optimization</a></li> <li><a href="/wiki/Cloud_storage" title="Cloud storage">Cloud storage</a></li> <li><a href="/wiki/Content-addressable_storage" title="Content-addressable storage">Content-addressable storage</a></li> <li><a href="/wiki/Convergent_encryption" title="Convergent encryption">Convergent encryption</a></li> <li><a href="/wiki/Delta_encoding" title="Delta encoding">Delta encoding</a></li> <li><a href="/wiki/Identity_resolution" class="mw-redirect" title="Identity resolution">Identity resolution</a></li> <li><a href="/wiki/Information_integration" title="Information integration">Information integration</a></li> <li><a href="/wiki/Linked_data" title="Linked data">Linked data</a></li> <li><a href="/wiki/Pointer_(computer_programming)" title="Pointer (computer programming)">Pointer (computer programming)</a></li> <li><a href="/wiki/Record_linkage" title="Record linkage">Record linkage</a></li> <li><a href="/wiki/Single-instance_storage" title="Single-instance storage">Single-instance storage</a></li></ul> </div> <div class="mw-heading mw-heading2"><h2 id="References">References</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=12" title="Edit section: References"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <style data-mw-deduplicate="TemplateStyles:r1239543626">.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em;"> <ol class="references"> <li id="cite_note-1"><span class="mw-cite-backlink"><b><a href="#cite_ref-1">^</a></b></span> <span class="reference-text"><style data-mw-deduplicate="TemplateStyles:r1238218222">.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("//upload.wikimedia.org/wikipedia/commons/6/65/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("//upload.wikimedia.org/wikipedia/commons/d/d6/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("//upload.wikimedia.org/wikipedia/commons/a/aa/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("//upload.wikimedia.org/wikipedia/commons/4/4c/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}</style><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20190806233329/https://www.druva.com/blog/understanding-data-deduplication/">"Understanding Data Deduplication"</a>. <i>Druva</i>. 2009-01-09. Archived from <a rel="nofollow" class="external text" href="https://www.druva.com/blog/understanding-data-deduplication/">the original</a> on 2019-08-06<span class="reference-accessdate">. Retrieved <span class="nowrap">2019-08-06</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=Druva&rft.atitle=Understanding+Data+Deduplication&rft.date=2009-01-09&rft_id=https%3A%2F%2Fwww.druva.com%2Fblog%2Funderstanding-data-deduplication%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-snia-2"><span class="mw-cite-backlink">^ <a href="#cite_ref-snia_2-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-snia_2-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.snia.org/education/online-dictionary/D">"SNIA Dictionary » Dictionary D"</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20181224023910/https://www.snia.org/education/dictionary/d">Archived</a> from the original on 2018-12-24<span class="reference-accessdate">. Retrieved <span class="nowrap">2023-12-06</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=SNIA+Dictionary+%C2%BB+Dictionary+D&rft_id=https%3A%2F%2Fwww.snia.org%2Feducation%2Fonline-dictionary%2FD&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-3"><span class="mw-cite-backlink"><b><a href="#cite_ref-3">^</a></b></span> <span class="reference-text"><a rel="nofollow" class="external text" href="https://www.techtarget.com/searchdatabackup/tip/Compression-deduplication-and-encryption-Whats-the-difference">Compression, deduplication and encryption: What's the difference?</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20181223211506/https://searchdatabackup.techtarget.com/tip/Compression-deduplication-and-encryption-Whats-the-difference">Archived</a> 2018-12-23 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>, Stephen Bigelow and Paul Crocetti</span> </li> <li id="cite_note-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-4">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://backupcentral.com/inline-or-post-process/">"In-line or post-process de-duplication? (updated 6-08)"</a>. Backup Central. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20091206035054/http://www.backupcentral.com/content/view/134/47">Archived</a> from the original on 2009-12-06<span class="reference-accessdate">. Retrieved <span class="nowrap">2023-12-06</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=In-line+or+post-process+de-duplication%3F+%28updated+6-08%29&rft.pub=Backup+Central&rft_id=https%3A%2F%2Fbackupcentral.com%2Finline-or-post-process%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-5"><span class="mw-cite-backlink"><b><a href="#cite_ref-5">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.techtarget.com/searchstorage/definition/data-deduplication">"Inline vs. post-processing deduplication appliances"</a>. techtarget.com. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20090609122811/http://searchdatabackup.techtarget.com/tip/0,289483,sid187_gci1315295,00.html">Archived</a> from the original on 2009-06-09<span class="reference-accessdate">. Retrieved <span class="nowrap">2023-12-06</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Inline+vs.+post-processing+deduplication+appliances&rft.pub=techtarget.com&rft_id=https%3A%2F%2Fwww.techtarget.com%2Fsearchstorage%2Fdefinition%2Fdata-deduplication&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-6">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20091004073508/http://www.microsoft.com/windowsserver2008/en/us/WSS08/SIS.aspx">"Windows Server 2008: Windows Storage Server 2008"</a>. Microsoft.com. Archived from <a rel="nofollow" class="external text" href="http://www.microsoft.com/windowsserver2008/en/us/WSS08/SIS.aspx">the original</a> on 2009-10-04<span class="reference-accessdate">. Retrieved <span class="nowrap">2009-10-16</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Windows+Server+2008%3A+Windows+Storage+Server+2008&rft.pub=Microsoft.com&rft_id=http%3A%2F%2Fwww.microsoft.com%2Fwindowsserver2008%2Fen%2Fus%2FWSS08%2FSIS.aspx&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://web.archive.org/web/20100206071701/http://www.netapp.com/us/products/platform-os/dedupe.html">"Products - Platform OS"</a>. NetApp. Archived from <a rel="nofollow" class="external text" href="http://www.netapp.com/us/products/platform-os/dedupe.html">the original</a> on 2010-02-06<span class="reference-accessdate">. Retrieved <span class="nowrap">2009-10-16</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&rft.genre=unknown&rft.btitle=Products+-+Platform+OS&rft.pub=NetApp&rft_id=http%3A%2F%2Fwww.netapp.com%2Fus%2Fproducts%2Fplatform-os%2Fdedupe.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://lwn.net/Articles/335380/">"The reflink(2) system call v5"</a>. <i>lwn.net</i>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20151002000848/http://lwn.net/Articles/335380/">Archived</a> from the original on 2015-10-02<span class="reference-accessdate">. Retrieved <span class="nowrap">2019-10-04</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=lwn.net&rft.atitle=The+reflink%282%29+system+call+v5.&rft_id=https%3A%2F%2Flwn.net%2FArticles%2F335380%2F&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://man7.org/linux/man-pages/man2/ioctl_ficlonerange.2.html">"ioctl_ficlonerange(2)"</a>. <i>Linux Manual Page</i>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20191007043757/http://man7.org/linux/man-pages/man2/ioctl_ficlonerange.2.html">Archived</a> from the original on 2019-10-07<span class="reference-accessdate">. Retrieved <span class="nowrap">2019-10-04</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=Linux+Manual+Page&rft.atitle=ioctl_ficlonerange%282%29&rft_id=https%3A%2F%2Fman7.org%2Flinux%2Fman-pages%2Fman2%2Fioctl_ficlonerange.2.html&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-10"><span class="mw-cite-backlink"><b><a href="#cite_ref-10">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFKazuki_MATSUDA" class="citation web cs1">Kazuki MATSUDA. <a rel="nofollow" class="external text" href="https://github.com/git-lfs/git-lfs/pull/3790">"Add clonefile on Windows over ReFS support"</a>. <i>GitHub</i>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20210113201013/https://github.com/git-lfs/git-lfs/pull/3790">Archived</a> from the original on 2021-01-13<span class="reference-accessdate">. Retrieved <span class="nowrap">2020-02-23</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=unknown&rft.jtitle=GitHub&rft.atitle=Add+clonefile+on+Windows+over+ReFS+support&rft.au=Kazuki+MATSUDA&rft_id=https%3A%2F%2Fgithub.com%2Fgit-lfs%2Fgit-lfs%2Fpull%2F3790&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> <li id="cite_note-11"><span class="mw-cite-backlink"><b><a href="#cite_ref-11">^</a></b></span> <span class="reference-text">An example of an implementation that checks for identity rather than assuming it is described in <a rel="nofollow" class="external text" href="http://appft1.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&Sect2=HITOFF&p=1&u=%2Fnetahtml%2FPTO%2Fsearch-bool.html&r=1&f=G&l=50&co1=AND&d=PG01&s1=shnelvar&OS=shnelvar&RS=shnelvar">"US Patent application # 20090307251"</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20170115031922/http://appft1.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&Sect2=HITOFF&p=1&u=%2Fnetahtml%2FPTO%2Fsearch-bool.html&r=1&f=G&l=50&co1=AND&d=PG01&s1=shnelvar&OS=shnelvar&RS=shnelvar">Archived</a> 2017-01-15 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>.</span> </li> <li id="cite_note-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-12">^</a></b></span> <span class="reference-text"><a rel="nofollow" class="external text" href="https://searchitchannel.techtarget.com/tip/Explaining-deduplication-rates-and-single-instance-storage-to-clients">Explaining deduplication rates and single-instance storage to clients</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20181223211511/https://searchitchannel.techtarget.com/tip/Explaining-deduplication-rates-and-single-instance-storage-to-clients">Archived</a> 2018-12-23 at the <a href="/wiki/Wayback_Machine" title="Wayback Machine">Wayback Machine</a>. George Crump, Storage Switzerland</span> </li> <li id="cite_note-13"><span class="mw-cite-backlink"><b><a href="#cite_ref-13">^</a></b></span> <span class="reference-text"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1238218222"><cite id="CITEREFCHRISTIAN_CACHINMATTHIAS_SCHUNTER2011" class="citation journal cs1">CHRISTIAN CACHIN; MATTHIAS SCHUNTER (December 2011). <a rel="nofollow" class="external text" href="https://spectrum.ieee.org/a-cloud-you-can-trust">"A Cloud You Can Trust"</a>. <i><a href="/wiki/IEEE_Spectrum" title="IEEE Spectrum">IEEE Spectrum</a></i>. <a href="/wiki/IEEE" class="mw-redirect" title="IEEE">IEEE</a>. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20120102010532/http://spectrum.ieee.org/computing/networks/a-cloud-you-can-trust">Archived</a> from the original on 2012-01-02<span class="reference-accessdate">. Retrieved <span class="nowrap">2011-12-21</span></span>.</cite><span title="ctx_ver=Z39.88-2004&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.jtitle=IEEE+Spectrum&rft.atitle=A+Cloud+You+Can+Trust&rft.date=2011-12&rft.au=CHRISTIAN+CACHIN&rft.au=MATTHIAS+SCHUNTER&rft_id=https%3A%2F%2Fspectrum.ieee.org%2Fa-cloud-you-can-trust&rfr_id=info%3Asid%2Fen.wikipedia.org%3AData+deduplication" class="Z3988"></span></span> </li> </ol></div> <div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=Data_deduplication&action=edit&section=13" title="Edit section: External links"><span>edit</span></a><span class="mw-editsection-bracket">]</span></span></div> <ul><li>Biggar, Heidi(2007.12.11). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20080516150913/http://www.infostor.com/webcast/display_webcast.cfm?id=540">WebCast: The Data Deduplication Effect</a></li> <li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20110928061457/http://www.tacoma.washington.edu/tech/docs/research/gradresearch/MSpiz.pdf">Using Latent Semantic Indexing for Data Deduplication</a>.</li> <li><a rel="nofollow" class="external text" href="https://www.forbes.com/2009/08/08/exagrid-storage-data-technology-cio-network-tape.html">A Better Way to Store Data</a>.</li> <li><a rel="nofollow" class="external text" href="https://www.eweek.com/database/what-is-the-difference-between-data-deduplication-file-deduplication-and-data-compression/">What Is the Difference Between Data Deduplication, File Deduplication, and Data Compression?</a> - Database from eWeek</li> <li><a rel="nofollow" class="external text" href="https://www.snia.org/forums/dmf/programs/data_protect_init/ddsrsig">SNIA DDSR SIG</a> <ul><li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20110409132151/http://www.snia.org/forums/dmf/knowledge/white_papers_and_reports/Understanding_Data_Deduplication_Ratios-20080718.pdf">Understanding Data Deduplication Ratios</a></li></ul></li> <li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20100911194757/http://www.itnext.in/content/doing-more-less.html">Doing More with Less by Jatinder Singh</a></li> <li><a rel="nofollow" class="external text" href="https://www.text-filter.com/tools/remove-duplicate-lines/">DeDuplication</a> Demo</li></ul> <div class="navbox-styles"><style data-mw-deduplicate="TemplateStyles:r1129693374">.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}</style><style data-mw-deduplicate="TemplateStyles:r1236075235">.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}</style></div><div role="navigation" class="navbox" aria-labelledby="File_systems" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r1129693374"><style data-mw-deduplicate="TemplateStyles:r1239400231">.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}</style><div class="navbar plainlinks hlist navbar-mini"><ul><li class="nv-view"><a href="/wiki/Template:File_systems" title="Template:File systems"><abbr title="View this template">v</abbr></a></li><li class="nv-talk"><a href="/wiki/Template_talk:File_systems" title="Template talk:File systems"><abbr title="Discuss this template">t</abbr></a></li><li class="nv-edit"><a href="/wiki/Special:EditPage/Template:File_systems" title="Special:EditPage/Template:File systems"><abbr title="Edit this template">e</abbr></a></li></ul></div><div id="File_systems" style="font-size:114%;margin:0 4em"><a href="/wiki/File_system" title="File system">File systems</a></div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div> <ul><li><a href="/wiki/Comparison_of_file_systems" title="Comparison of file systems">Comparison of file systems</a> <ul><li><a href="/wiki/Comparison_of_distributed_file_systems" title="Comparison of distributed file systems">distributed</a></li></ul></li> <li><a href="/wiki/Unix_filesystem" title="Unix filesystem">Unix filesystem</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Disk and<br />non-rotating</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Advanced_Disc_Filing_System" title="Advanced Disc Filing System">ADFS</a></li> <li><a href="/wiki/AdvFS" title="AdvFS">AdvFS</a></li> <li><a href="/wiki/Amiga_Fast_File_System" title="Amiga Fast File System">Amiga FFS</a></li> <li><a href="/wiki/Amiga_Old_File_System" title="Amiga Old File System">Amiga OFS</a></li> <li><a href="/wiki/Apple_File_System" title="Apple File System">APFS</a></li> <li><a href="/wiki/AtheOS_File_System" class="mw-redirect" title="AtheOS File System">AthFS</a></li> <li><a href="/wiki/Bcachefs" title="Bcachefs">bcachefs</a></li> <li>BFS <ul><li><a href="/wiki/Be_File_System" title="Be File System">Be File System</a></li> <li><a href="/wiki/Boot_File_System" title="Boot File System">Boot File System</a></li> <li>Byte File System (<a href="/wiki/Z/VM" title="Z/VM">z/VM</a>)</li></ul></li> <li><a href="/wiki/Btrfs" title="Btrfs">Btrfs</a></li> <li><a href="/wiki/StorNext_File_System" title="StorNext File System">CVFS</a></li> <li><a href="/wiki/CXFS" title="CXFS">CXFS</a></li> <li><a href="/wiki/Disc_Filing_System" title="Disc Filing System">DFS</a></li> <li>EFS <ul><li><a href="/wiki/Encrypting_File_System" title="Encrypting File System">Encrypting File System</a></li> <li><a href="/wiki/Extent_File_System" title="Extent File System">Extent File System</a></li></ul></li> <li><a href="/wiki/Episode_filesystem" title="Episode filesystem">Episode</a></li> <li><a href="/wiki/Extended_file_system" title="Extended file system">ext</a> <ul><li><a href="/wiki/Ext2" title="Ext2">ext2</a></li> <li><a href="/wiki/Ext3" title="Ext3">ext3</a></li> <li><a href="/wiki/Ext3cow" title="Ext3cow">ext3cow</a></li> <li><a href="/wiki/Ext4" title="Ext4">ext4</a></li></ul></li> <li><a href="/wiki/File_Allocation_Table" title="File Allocation Table">FAT</a> <ul><li><a href="/wiki/ExFAT" title="ExFAT">exFAT</a></li></ul></li> <li><a href="/wiki/Files-11" title="Files-11">Files-11</a></li> <li><a href="/wiki/Fossil_(file_system)" title="Fossil (file system)">Fossil</a></li> <li><a href="/wiki/GPFS" title="GPFS">GPFS</a></li> <li><a href="/wiki/HAMMER_(file_system)" title="HAMMER (file system)">HAMMER</a> <ul><li><a href="/wiki/HAMMER2" title="HAMMER2">HAMMER2</a></li></ul></li> <li><a href="/wiki/Hierarchical_File_System_(Apple)" title="Hierarchical File System (Apple)">HFS</a> (<a href="/wiki/Classic_Mac_OS" title="Classic Mac OS">Classic Mac OS</a>)</li> <li><a href="/wiki/Hierarchical_File_System_(IBM_MVS)" title="Hierarchical File System (IBM MVS)">HFS</a> (<a href="/wiki/MVS" title="MVS">MVS</a>)</li> <li><a href="/wiki/HFS_Plus" title="HFS Plus">HFS+</a></li> <li><a href="/wiki/High_Performance_File_System" title="High Performance File System">HPFS</a></li> <li><a href="/wiki/High_Throughput_File_System" title="High Throughput File System">HTFS</a></li> <li><a href="/wiki/JFS_(file_system)" title="JFS (file system)">JFS</a></li> <li><a href="/wiki/Log-structured_File_System_(BSD)" title="Log-structured File System (BSD)">LFS</a></li> <li>MFS <ul><li><a href="/wiki/Macintosh_File_System" title="Macintosh File System">Macintosh File System</a></li> <li><a href="/wiki/TiVo_Media_File_System" title="TiVo Media File System">TiVo Media File System</a></li></ul></li> <li><a href="/wiki/MINIX_file_system" title="MINIX file system">MINIX</a></li> <li><a href="/wiki/NetWare_File_System" title="NetWare File System">NetWare File System</a></li> <li><a href="/wiki/Next3" title="Next3">Next3</a></li> <li><a href="/wiki/NILFS" title="NILFS">NILFS</a> <ul><li><a href="/wiki/NILFS" title="NILFS">NILFS2</a></li></ul></li> <li><a href="/wiki/Novell_Storage_Services" title="Novell Storage Services">NSS</a></li> <li><a href="/wiki/NTFS" title="NTFS">NTFS</a></li> <li><a href="/wiki/OneFS_distributed_file_system" title="OneFS distributed file system">OneFS</a></li> <li><a href="/wiki/OpenZFS" title="OpenZFS">OpenZFS</a></li> <li><a href="/wiki/Professional_File_System" title="Professional File System">PFS</a></li> <li><a href="/wiki/QFS" title="QFS">QFS</a></li> <li><a href="/wiki/QNX4FS" title="QNX4FS">QNX4FS</a></li> <li><a href="/wiki/ReFS" title="ReFS">ReFS</a></li> <li><a href="/wiki/ReiserFS" title="ReiserFS">ReiserFS</a> <ul><li><a href="/wiki/Reiser4" title="Reiser4">Reiser4</a></li></ul></li> <li><a href="/wiki/Datalight#Reliance_family" title="Datalight">Reliance</a></li> <li><a href="/wiki/Datalight#Reliance_Nitro" title="Datalight">Reliance Nitro</a></li> <li><a href="/wiki/Remote_File_Sharing" title="Remote File Sharing">RFS</a></li> <li>SFS <ul><li>Shared File System (<a href="/wiki/VM_(operating_system)" title="VM (operating system)">VM</a>)</li> <li><a href="/wiki/Smart_File_System" title="Smart File System">Smart File System</a></li></ul></li> <li><a href="/wiki/StorNext_File_System" title="StorNext File System">SNFS</a></li> <li><a href="/wiki/Soup_(Apple)" title="Soup (Apple)">Soup (Apple)</a></li> <li><a href="/wiki/Tux3" title="Tux3">Tux3</a></li> <li><a href="/wiki/UBIFS" title="UBIFS">UBIFS</a></li> <li><a href="/wiki/Unix_File_System" title="Unix File System">UFS/UFS2</a> <ul><li><a href="/wiki/Soft_updates" title="Soft updates">soft updates</a></li> <li><a href="/wiki/Write_Ahead_Physical_Block_Logging" title="Write Ahead Physical Block Logging">WAPBL</a></li></ul></li> <li><a href="/wiki/Veritas_File_System" title="Veritas File System">VxFS</a></li> <li><a href="/wiki/Write_Anywhere_File_Layout" title="Write Anywhere File Layout">WAFL</a></li> <li><a href="/wiki/Xiafs" title="Xiafs">Xiafs</a></li> <li><a href="/wiki/XFS" title="XFS">XFS</a></li> <li><a href="/wiki/Xsan" title="Xsan">Xsan</a></li> <li><a href="/wiki/ZFS_(z/OS_file_system)" title="ZFS (z/OS file system)">zFS</a> (<a href="/wiki/Z/OS" title="Z/OS">z/OS</a>)</li> <li><a href="/wiki/ZFS" title="ZFS">ZFS</a> (<a href="/wiki/Sun_Microsystems" title="Sun Microsystems">Sun</a>)</li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Optical_disc" title="Optical disc">Optical disc</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/High_Sierra_Format" class="mw-redirect" title="High Sierra Format">HSF</a></li> <li><a href="/wiki/ISO_9660" title="ISO 9660">ISO 9660</a></li> <li><a href="/wiki/ISO_13490" title="ISO 13490">ISO 13490</a></li> <li><a href="/wiki/Universal_Disk_Format" title="Universal Disk Format">UDF</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Flash_memory" title="Flash memory">Flash memory</a> and <a href="/wiki/Solid-state_drive" title="Solid-state drive">SSD</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Apple_File_System" title="Apple File System">APFS</a></li> <li><a href="/wiki/File_Allocation_Table" title="File Allocation Table">FAT</a></li> <li><a href="/wiki/ExFAT" title="ExFAT">exFAT</a></li> <li><a href="/wiki/Transaction-Safe_FAT_File_System" title="Transaction-Safe FAT File System">TFAT</a></li> <li><a href="/wiki/EROFS" title="EROFS">EROFS</a></li> <li><a href="/wiki/F2FS" title="F2FS">F2FS</a></li> <li><a href="/wiki/JFS_(file_system)" title="JFS (file system)">JFS</a></li> <li><a href="/wiki/Non-Volatile_File_System" title="Non-Volatile File System">NVFS</a></li></ul> </div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th id="host-side_wear_leveling" scope="row" class="navbox-group" style="width:1%">host-side <a href="/wiki/Wear_leveling" title="Wear leveling">wear leveling</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/CHFS" title="CHFS">CHFS</a></li> <li><a href="/wiki/JFFS" title="JFFS">JFFS</a></li> <li><a href="/wiki/JFFS2" title="JFFS2">JFFS2</a></li> <li><a href="/wiki/LogFS" title="LogFS">LogFS</a></li> <li><a href="/wiki/NILFS" title="NILFS">NILFS</a> <ul><li><a href="/wiki/NILFS" title="NILFS">NILFS2</a></li></ul></li> <li><a href="/wiki/YAFFS" title="YAFFS">YAFFS</a></li> <li><a href="/wiki/UBIFS" title="UBIFS">UBIFS</a></li></ul> </div></td></tr></tbody></table><div> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Clustered_file_system#Distributed_file_systems" title="Clustered file system">Distributed parallel</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/BeeGFS" title="BeeGFS">BeeGFS</a></li> <li><a href="/wiki/Ceph_(software)" title="Ceph (software)">Ceph</a></li> <li><a href="/wiki/CXFS" title="CXFS">CXFS</a></li> <li><a href="/wiki/GFS2" title="GFS2">GFS2</a></li> <li><a href="/wiki/Google_File_System" title="Google File System">Google File System</a></li> <li><a href="/wiki/OCFS2" title="OCFS2">OCFS2</a></li> <li><a href="/wiki/OrangeFS" title="OrangeFS">OrangeFS</a></li> <li><a href="/wiki/Parallel_Virtual_File_System" title="Parallel Virtual File System">PVFS</a></li> <li><a href="/wiki/Quantcast_File_System" title="Quantcast File System">QFS</a></li> <li><a href="/wiki/Xsan" title="Xsan">Xsan</a></li> <li><i><a href="/wiki/List_of_file_systems#Distributed_parallel_file_systems" title="List of file systems">more...</a></i></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Network-attached_storage" title="Network-attached storage">NAS</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/9P_(protocol)" title="9P (protocol)">9P</a></li> <li><a href="/wiki/Andrew_File_System" title="Andrew File System">AFS</a> (<a href="/wiki/OpenAFS" title="OpenAFS">OpenAFS</a>)</li> <li><a href="/wiki/Apple_Filing_Protocol" title="Apple Filing Protocol">AFP</a></li> <li><a href="/wiki/Coda_(file_system)" title="Coda (file system)">Coda</a></li> <li><a href="/wiki/Distributed_File_System_(Microsoft)" title="Distributed File System (Microsoft)">DFS</a></li> <li><a href="/wiki/Google_File_System" title="Google File System">Google File System</a></li> <li><a href="/wiki/GPFS" title="GPFS">GPFS</a></li> <li><a href="/wiki/Lustre_(file_system)" title="Lustre (file system)">Lustre</a></li> <li><a href="/wiki/NetWare_Core_Protocol" title="NetWare Core Protocol">NCP</a></li> <li><a href="/wiki/Network_File_System" title="Network File System">NFS</a></li> <li><a href="/wiki/Elliptics" title="Elliptics">POHMELFS</a></li> <li><a href="/wiki/Apache_Hadoop" title="Apache Hadoop">Hadoop</a></li> <li><a href="/wiki/Server_Message_Block" title="Server Message Block">SMB (CIFS)</a></li> <li><a href="/wiki/SSHFS" title="SSHFS">SSHFS</a></li> <li><i><a href="/wiki/List_of_file_systems#Distributed_file_systems" title="List of file systems">more...</a></i></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Specialized</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Aufs" title="Aufs">Aufs</a></li> <li><a href="/wiki/AXFS" title="AXFS">AXFS</a></li> <li><a href="/wiki/Boot_File_System" title="Boot File System">Boot File System</a></li> <li><a href="/wiki/CDfs" title="CDfs">CDfs</a></li> <li><a href="/wiki/Compact_Disc_File_System" title="Compact Disc File System">Compact Disc File System</a></li> <li><a href="/wiki/Cramfs" title="Cramfs">cramfs</a></li> <li><a href="/wiki/Davfs2" title="Davfs2">Davfs2</a></li> <li><a href="/wiki/EROFS" title="EROFS">EROFS</a></li> <li><a href="/wiki/FTPFS" title="FTPFS">FTPFS</a></li> <li><a href="/wiki/Filesystem_in_Userspace" title="Filesystem in Userspace">FUSE</a></li> <li><a href="/wiki/Lnfs" title="Lnfs">Lnfs</a></li> <li><a href="/wiki/Linear_Tape_File_System" title="Linear Tape File System">LTFS</a></li> <li><a href="/wiki/NOVA_(filesystem)" title="NOVA (filesystem)">NOVA</a></li> <li><a href="/wiki/Rational_MultiVersion_File_System" class="mw-redirect" title="Rational MultiVersion File System">MVFS</a></li> <li><a href="/wiki/SquashFS" title="SquashFS">SquashFS</a></li> <li><a href="/wiki/FAT_filesystem_and_Linux" title="FAT filesystem and Linux">UMSDOS</a></li> <li><a href="/wiki/OverlayFS" title="OverlayFS">OverlayFS</a></li> <li><a href="/wiki/UnionFS" title="UnionFS">UnionFS</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Pseudo</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Configfs" title="Configfs">configfs</a></li> <li><a href="/wiki/Device_file" title="Device file">devfs</a></li> <li><a href="/wiki/Debugfs" title="Debugfs">debugfs</a></li> <li><a href="/wiki/Kernfs_(BSD)" title="Kernfs (BSD)">kernfs</a></li> <li><a href="/wiki/Procfs" title="Procfs">procfs</a></li> <li>specfs</li> <li><a href="/wiki/Sysfs" title="Sysfs">sysfs</a></li> <li><a href="/wiki/Tmpfs" title="Tmpfs">tmpfs</a></li> <li><a href="/wiki/WinFS" title="WinFS">WinFS</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Filesystem-level_encryption" title="Filesystem-level encryption">Encrypted</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/ECryptfs" title="ECryptfs">eCryptfs</a></li> <li><a href="/wiki/EncFS" title="EncFS">EncFS</a></li> <li><a href="/wiki/Encrypting_File_System" title="Encrypting File System">EFS</a></li> <li><a href="/wiki/Rubberhose_(file_system)" class="mw-redirect" title="Rubberhose (file system)">Rubberhose</a></li> <li><a href="/wiki/SSHFS" title="SSHFS">SSHFS</a></li> <li><a href="/wiki/Oracle_ZFS" title="Oracle ZFS">ZFS</a></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Types</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Clustered_file_system" title="Clustered file system">Clustered</a> <ul><li><a href="/wiki/Global_file_system" title="Global file system">Global</a></li> <li><a href="/wiki/Grid_file_system" title="Grid file system">Grid</a></li> <li><a href="/wiki/Self-certifying_File_System" title="Self-certifying File System">Self-certifying</a></li></ul></li> <li><a href="/wiki/Flash_file_system" title="Flash file system">Flash</a></li> <li><a href="/wiki/Journaling_file_system" title="Journaling file system">Journaling</a></li> <li><a href="/wiki/Log-structured_file_system" title="Log-structured file system">Log-structured</a></li> <li><a href="/wiki/Object_storage" title="Object storage">Object</a></li> <li><a href="/wiki/Record-oriented_filesystem" title="Record-oriented filesystem">Record-oriented</a></li> <li><a href="/wiki/Semantic_file_system" title="Semantic file system">Semantic</a></li> <li><a href="/wiki/Steganographic_file_system" title="Steganographic file system">Steganographic</a></li> <li><a href="/wiki/Synthetic_file_system" title="Synthetic file system">Synthetic</a></li> <li><a href="/wiki/Versioning_file_system" title="Versioning file system">Versioning</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Features</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><td colspan="2" class="navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Case_preservation" title="Case preservation">Case preservation</a></li> <li><a href="/wiki/Copy-on-write" title="Copy-on-write">Copy-on-write</a></li> <li><a class="mw-selflink selflink">Data deduplication</a></li> <li><a href="/wiki/Data_scrubbing" title="Data scrubbing">Data scrubbing</a></li> <li><a href="/wiki/Execute_in_place" title="Execute in place">Execute in place</a></li> <li><a href="/wiki/Extent_(file_systems)" title="Extent (file systems)">Extent</a></li> <li><a href="/wiki/File_attribute" title="File attribute">File attribute</a> <ul><li><a href="/wiki/Extended_file_attributes" title="Extended file attributes">Extended file attributes</a></li></ul></li> <li><a href="/wiki/File_change_log" title="File change log">File change log</a></li> <li><a href="/wiki/Fork_(file_system)" title="Fork (file system)">Fork</a></li> <li>Links <ul><li><a href="/wiki/Hard_link" title="Hard link">Hard</a></li> <li><a href="/wiki/Symbolic_link" title="Symbolic link">Symbolic</a></li></ul></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Computer_access_control" title="Computer access control">Access control</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Access-control_list" title="Access-control list">Access-control list</a></li> <li><a href="/wiki/Filesystem-level_encryption" title="Filesystem-level encryption">Filesystem-level encryption</a></li> <li><a href="/wiki/File-system_permissions" title="File-system permissions">Permissions</a> <ul><li><a href="/wiki/Chmod" title="Chmod">Modes</a></li> <li><a href="/wiki/Sticky_bit" title="Sticky bit">Sticky bit</a></li></ul></li></ul> </div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/Interface_(computing)" title="Interface (computing)">Interfaces</a></th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/File_manager" title="File manager">File manager</a></li> <li><a href="/wiki/File_system_API" title="File system API">File system API</a> <ul><li><a href="/wiki/Installable_File_System" title="Installable File System">Installable File System</a></li></ul></li> <li><a href="/wiki/Virtual_file_system" title="Virtual file system">Virtual file system</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%"><a href="/wiki/List_of_file_systems" title="List of file systems">Lists</a></th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/List_of_cryptographic_file_systems" title="List of cryptographic file systems">Cryptographic</a></li> <li><a href="/wiki/List_of_default_file_systems" title="List of default file systems">Default</a></li> <li><a href="/wiki/List_of_log-structured_file_systems" title="List of log-structured file systems">Log-structured</a></li></ul> </div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Layouts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"> <ul><li><a href="/wiki/Master_Boot_Record" class="mw-redirect" title="Master Boot Record">Master Boot Record</a></li> <li><a href="/wiki/GUID_Partition_Table" title="GUID Partition Table">GUID Partition Table</a></li> <li><a href="/wiki/Apple_Partition_Map" title="Apple Partition Map">Apple Partition Map</a></li></ul> </div></td></tr></tbody></table></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://en.wikipedia.org/w/index.php?title=Data_deduplication&oldid=1232070507">https://en.wikipedia.org/w/index.php?title=Data_deduplication&oldid=1232070507</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Help:Category" title="Help:Category">Categories</a>: <ul><li><a href="/wiki/Category:Data_compression" title="Category:Data compression">Data compression</a></li><li><a href="/wiki/Category:Data_management" title="Category:Data management">Data management</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Hidden categories: <ul><li><a href="/wiki/Category:Webarchive_template_wayback_links" title="Category:Webarchive template wayback links">Webarchive template wayback links</a></li><li><a href="/wiki/Category:Articles_with_short_description" title="Category:Articles with short description">Articles with short description</a></li><li><a href="/wiki/Category:Short_description_matches_Wikidata" title="Category:Short description matches Wikidata">Short description matches Wikidata</a></li><li><a href="/wiki/Category:All_articles_with_unsourced_statements" title="Category:All articles with unsourced statements">All articles with unsourced statements</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_August_2016" title="Category:Articles with unsourced statements from August 2016">Articles with unsourced statements from August 2016</a></li><li><a href="/wiki/Category:Articles_with_unsourced_statements_from_December_2023" title="Category:Articles with unsourced statements from December 2023">Articles with unsourced statements from December 2023</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 1 July 2024, at 19:57<span class="anonymous-show"> (UTC)</span>.</li> <li id="footer-info-copyright">Text is available under the <a href="/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License" title="Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License">Creative Commons Attribution-ShareAlike 4.0 License</a>; additional terms may apply. By using this site, you agree to the <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use" class="extiw" title="foundation:Special:MyLanguage/Policy:Terms of Use">Terms of Use</a> and <a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy" class="extiw" title="foundation:Special:MyLanguage/Policy:Privacy policy">Privacy Policy</a>. Wikipedia® is a registered trademark of the <a rel="nofollow" class="external text" href="https://wikimediafoundation.org/">Wikimedia Foundation, Inc.</a>, a non-profit organization.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Wikipedia:About">About Wikipedia</a></li> <li id="footer-places-disclaimers"><a href="/wiki/Wikipedia:General_disclaimer">Disclaimers</a></li> <li id="footer-places-contact"><a href="//en.wikipedia.org/wiki/Wikipedia:Contact_us">Contact Wikipedia</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Universal_Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/en.wikipedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//en.m.wikipedia.org/w/index.php?title=Data_deduplication&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-7ztkk","wgBackendResponseTime":187,"wgPageParseReport":{"limitreport":{"cputime":"0.379","walltime":"0.498","ppvisitednodes":{"value":1435,"limit":1000000},"postexpandincludesize":{"value":67696,"limit":2097152},"templateargumentsize":{"value":2118,"limit":2097152},"expansiondepth":{"value":12,"limit":100},"expensivefunctioncount":{"value":3,"limit":500},"unstrip-depth":{"value":1,"limit":20},"unstrip-size":{"value":46548,"limit":5000000},"entityaccesscount":{"value":0,"limit":400},"timingprofile":["100.00% 413.729 1 -total"," 39.58% 163.755 1 Template:Reflist"," 30.54% 126.347 9 Template:Cite_web"," 21.89% 90.553 5 Template:Navbox"," 21.29% 88.091 1 Template:File_systems"," 19.75% 81.731 1 Template:Short_description"," 12.48% 51.641 2 Template:Citation_needed"," 11.08% 45.847 2 Template:Pagetype"," 10.73% 44.401 2 Template:Fix"," 6.91% 28.573 4 Template:Category_handler"]},"scribunto":{"limitreport-timeusage":{"value":"0.224","limit":"10.000"},"limitreport-memusage":{"value":5423874,"limit":52428800}},"cachereport":{"origin":"mw-web.codfw.main-f69cdc8f6-jlfln","timestamp":"20241122150634","ttl":2592000,"transientcontent":false}}});});</script> <script type="application/ld+json">{"@context":"https:\/\/schema.org","@type":"Article","name":"Data deduplication","url":"https:\/\/en.wikipedia.org\/wiki\/Data_deduplication","sameAs":"http:\/\/www.wikidata.org\/entity\/Q1182260","mainEntity":"http:\/\/www.wikidata.org\/entity\/Q1182260","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\/\/www.wikimedia.org\/static\/images\/wmf-hor-googpub.png"}},"datePublished":"2008-04-29T07:14:18Z","dateModified":"2024-07-01T19:57:07Z","headline":"data processing technique to eliminate duplicate copies of repeating data"}</script> </body> </html>