CINXE.COM

<!DOCTYPE html> <html class="client-nojs mf-expand-sections-clientpref-0 mf-font-size-clientpref-small mw-mf-amc-clientpref-0" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Data Platform Engineering - Wikitech</title> <script>(function(){var className="client-js mf-expand-sections-clientpref-0 mf-font-size-clientpref-small mw-mf-amc-clientpref-0";var cookie=document.cookie.match(/(?:^|; )labswikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"6ad54032-176f-455c-ae8c-51f511c6ad67","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Data_Platform_Engineering","wgTitle":"Data Platform Engineering","wgCurRevisionId":2204281,"wgRevisionId":2204281,"wgArticleId":453836, "wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Data_Platform_Engineering","wgRelevantArticleId":453836,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikitech","wgCiteReferencePreviewsActive":false,"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFMode":"stable","wgMFAmc":false,"wgMFAmcOutreachActive":false,"wgMFAmcOutreachUserEligible":false,"wgMFLazyLoadImages":true,"wgMFEditNoticesFeatureConflict":false,"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgMFIsSupportedEditRequest":true,"wgMFScriptPath":"","wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":8000, "wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgDiscussionToolsFeaturesEnabled":{"replytool":true,"newtopictool":true,"sourcemodetoolbar":true,"topicsubscription":false,"autotopicsub":false,"visualenhancements":false,"visualenhancements_reply":false,"visualenhancements_pageframe":false},"wgDiscussionToolsFallbackEditMode":"visual","wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"wgMinervaPermissions":{"watchable":true,"watch":false},"wgMinervaFeatures":{"beta":false,"donate":true,"mobileOptionsLink":true,"categories":false,"pageIssues":true,"talkAtTop":false,"historyInPageActions":false,"overflowSubmenu":false,"tabsOnSpecials":true,"personalMenu":false,"mainMenuExpanded":false,"echo":true,"nightMode":false},"wgMinervaDownloadNamespaces":[0],"wgSiteNoticeId":"2.0"};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready", "ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.discussionTools.init.styles":"ready","oojs-ui-core.styles":"ready","oojs-ui.styles.indicators":"ready","mediawiki.widgets.styles":"ready","oojs-ui-core.icons":"ready","skins.minerva.styles":"ready","skins.minerva.content.styles.images":"ready","mediawiki.hlist":"ready","skins.minerva.codex.styles":"ready","skins.minerva.icons":"ready","ext.wikimediamessages.styles":"ready","mobile.init.styles":"ready","ext.dismissableSiteNotice.styles":"ready"};RLPAGEMODULES=["site","mediawiki.page.ready","skins.minerva.scripts","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.site","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mobile.init","ext.echo.centralauth","ext.discussionTools.init","ext.eventLogging","ext.wikimediaEvents","ext.checkUser.clientHints","ext.dismissableSiteNotice"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.discussionTools.init.styles%7Cext.dismissableSiteNotice.styles%7Cext.wikimediamessages.styles%7Cmediawiki.hlist%7Cmediawiki.widgets.styles%7Cmobile.init.styles%7Coojs-ui-core.icons%2Cstyles%7Coojs-ui.styles.indicators%7Cskins.minerva.codex.styles%7Cskins.minerva.content.styles.images%7Cskins.minerva.icons%2Cstyles&only=styles&skin=minerva"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=minerva"></script> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="theme-color" content="#eaecf0"> <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0"> <meta property="og:title" content="Data Platform Engineering - Wikitech"> <meta property="og:type" content="website"> <link rel="manifest" href="/w/api.php?action=webapp-manifest"> <link rel="icon" href="/static/favicon/wikitech.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikitech (en)"> <link rel="EditURI" type="application/rsd+xml" href="//wikitech.wikimedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://wikitech.wikimedia.org/wiki/Data_Platform_Engineering"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="ext-discussiontools-replytool-enabled ext-discussiontools-newtopictool-enabled ext-discussiontools-sourcemodetoolbar-enabled mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Data_Platform_Engineering rootpage-Data_Platform_Engineering stable issues-group-B skin-minerva action-view skin--responsive mw-mf-amc-disabled mw-mf"><div id="mw-mf-viewport"> <div id="mw-mf-page-center"> <a class="mw-mf-page-center__mask" href="#"></a> <header class="header-container header-chrome"> <div class="minerva-header"> <nav class="navigation-drawer toggle-list view-border-box"> <input type="checkbox" id="main-menu-input" class="toggle-list__checkbox" role="button" aria-haspopup="true" aria-expanded="false" aria-labelledby="mw-mf-main-menu-button"> <label role="button" for="main-menu-input" id="mw-mf-main-menu-button" aria-hidden="true" data-event-name="ui.mainmenu" class="cdx-button cdx-button--size-large cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--icon-only cdx-button--weight-quiet toggle-list__toggle"> <span class="minerva-icon minerva-icon--menu"></span> <span></span> </label> <div id="mw-mf-page-left" class="menu view-border-box"> <ul id="p-navigation" class="toggle-list__list"> <li class="toggle-list-item "> <a class="toggle-list-item__anchor menu__item--home" href="/wiki/Main_Page" data-mw="interface"> <span class="minerva-icon minerva-icon--home"></span> <span class="toggle-list-item__label">Home</span> </a> </li> <li class="toggle-list-item "> <a class="toggle-list-item__anchor menu__item--random" href="/wiki/Special:Random" data-mw="interface"> <span class="minerva-icon minerva-icon--die"></span> <span class="toggle-list-item__label">Random</span> </a> </li> <li class="toggle-list-item skin-minerva-list-item-jsonly"> <a class="toggle-list-item__anchor menu__item--nearby" href="/wiki/Special:Nearby" data-event-name="menu.nearby" data-mw="interface"> <span class="minerva-icon minerva-icon--mapPin"></span> <span class="toggle-list-item__label">Nearby</span> </a> </li> </ul> <ul id="p-personal" class="toggle-list__list"> <li class="toggle-list-item "> <a class="toggle-list-item__anchor menu__item--login" href="/w/index.php?title=Special:UserLogin&returnto=Data+Platform+Engineering&returntoquery=mobileaction%3Dtoggle_view_mobile" data-event-name="menu.login" data-mw="interface"> <span class="minerva-icon minerva-icon--logIn"></span> <span class="toggle-list-item__label">Log in</span> </a> </li> </ul> <ul id="pt-preferences" class="toggle-list__list"> <li class="toggle-list-item skin-minerva-list-item-jsonly"> <a class="toggle-list-item__anchor menu__item--settings" href="/w/index.php?title=Special:MobileOptions&returnto=Data+Platform+Engineering" data-event-name="menu.settings" data-mw="interface"> <span class="minerva-icon minerva-icon--settings"></span> <span class="toggle-list-item__label">Settings</span> </a> </li> </ul> <ul id="p-donation" class="toggle-list__list"> <li class="toggle-list-item "> <a class="toggle-list-item__anchor menu__item--donate" href="https://donate.wikimedia.org/?utm_source=donate&utm_medium=sidebar&utm_campaign=spontaneous&uselang=en&utm_key=minerva" data-event-name="menu.donate" data-mw="interface"> <span class="minerva-icon minerva-icon--heart"></span> <span class="toggle-list-item__label">Donate</span> </a> </li> </ul> <ul class="hlist"> <li class="toggle-list-item "> <a class="toggle-list-item__anchor menu__item--about" href="/wiki/Main_Page" data-mw="interface"> <span class="toggle-list-item__label">About Wikitech</span> </a> </li> <li class="toggle-list-item "> <a class="toggle-list-item__anchor menu__item--disclaimers" href="https://foundation.wikimedia.org/wiki/General_disclaimer" data-mw="interface"> <span class="toggle-list-item__label">Disclaimers</span> </a> </li> </ul> </div> <label class="main-menu-mask" for="main-menu-input"></label> </nav> <div class="branding-box"> <a href="/wiki/Main_Page"> <span><img src="/static/images/mobile/copyright/wikitech-wordmark.svg" alt="Wikitech" width="140" height="27" style="width: 8.75em; height: 1.6875em;"/> </span> </a> </div> <form action="/w/index.php" method="get" class="minerva-search-form"> <div class="search-box"> <input type="hidden" name="title" value="Special:Search"/> <input class="search skin-minerva-search-trigger" id="searchInput" type="search" name="search" placeholder="Search Wikitech" aria-label="Search Wikitech" autocapitalize="sentences" title="Search Wikitech [f]" accesskey="f"> <span class="search-box-icon-overlay"><span class="minerva-icon minerva-icon--search"></span> </span> </div> <button id="searchIcon" class="cdx-button cdx-button--size-large cdx-button--icon-only cdx-button--weight-quiet skin-minerva-search-trigger"> <span class="minerva-icon minerva-icon--search"></span> <span>Search</span> </button> </form> <nav class="minerva-user-navigation" aria-label="User navigation"> </nav> </div> </header> <main id="content" class="mw-body"> <div class="banner-container"> <div id="siteNotice"></div> </div> <div class="pre-content heading-holder"> <div class="page-heading"> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Data Platform Engineering</span></h1> <div class="tagline"></div> </div> <nav class="page-actions-menu"> <ul id="p-views" class="page-actions-menu__list"> <li id="language-selector" class="page-actions-menu__list-item"> <a role="button" href="" data-mw="interface" data-event-name="menu.languages" title="Language" class="cdx-button cdx-button--size-large cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--icon-only cdx-button--weight-quiet language-selector disabled"> <span class="minerva-icon minerva-icon--language"></span> <span>Language</span> </a> </li> <li id="page-actions-watch" class="page-actions-menu__list-item"> <a role="button" id="ca-watch" href="/w/index.php?title=Special:UserLogin&returnto=Data+Platform+Engineering" data-event-name="menu.watch" class="cdx-button cdx-button--size-large cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--icon-only cdx-button--weight-quiet menu__item--page-actions-watch"> <span class="minerva-icon minerva-icon--star"></span> <span>Watch</span> </a> </li> <li id="page-actions-viewsource" class="page-actions-menu__list-item"> <a role="button" id="ca-edit" href="/w/index.php?title=Data_Platform_Engineering&action=edit" data-event-name="menu.viewsource" data-mw="interface" class="cdx-button cdx-button--size-large cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--icon-only cdx-button--weight-quiet edit-page menu__item--page-actions-viewsource"> <span class="minerva-icon minerva-icon--editLock"></span> <span>View source</span> </a> </li> </ul> </nav>  <div id="mw-content-subtitle"></div> </div> <div id="bodyContent" class="content"> <div id="mw-content-text" class="mw-body-content"><script>function mfTempOpenSection(id){var block=document.getElementById("mf-section-"+id);block.className+=" open-block";block.previousSibling.className+=" open-block";}</script><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><section class="mf-section-0" id="mf-section-0"><p><br> The infrastructure and services maintained by the <a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering" class="extiw" title="mw:Data Platform Engineering">Data Platform Engineering team</a> support data producers and consumers in collecting, discovering, and using trustworthy data to derive data insights, conduct research and build new data products. </p> <style data-mw-deduplicate="TemplateStyles:r2126319">.mw-parser-output .tpl-contentgrid{margin:1em 0;display:grid}.mw-parser-output .tpl-contentgrid .mw-tpl-colorbox{margin:0}</style> <div class="tpl-contentgrid" style="grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); grid-gap: 2em; grid-auto-rows: minmax(100px, auto); width: auto;"><div class="mw-tpl-colorbox" style=""><style data-mw-deduplicate="TemplateStyles:r2199608">.mw-parser-output .mw-tpl-colorbox{box-sizing:border-box;margin:0.5em 0.5em 1em 0.5em;border-radius:2px;overflow:hidden;background:var(--background-color-base,#fff);color:var(--color-base,#202122);border:1px solid var(--border-color-base,#a2a9b1);box-shadow:0 2px 2px rgba(0,0,0,0.2)}.mw-parser-output .mw-tpl-colorbox-title{background:var(--background-color-interactive,#eaecf0);color:var(--color-emphasized,#000000);display:flex;gap:0.5rem;padding-top:0.4rem;padding-bottom:0.4rem}.mw-parser-output .mw-tpl-colorbox-title--linked:hover{background:var(--background-color-progressive-subtle,#eaf3ff)}.mw-parser-output .mw-tpl-colorbox-title>*:first-child{padding-left:1rem}.mw-parser-output .mw-tpl-colorbox-title>*:last-child{padding-right:1rem}.mw-parser-output .mw-tpl-colorbox-title>strong,.mw-parser-output .mw-tpl-colorbox-title>a{flex-grow:1}.mw-parser-output .mw-tpl-colorbox-title-icon{opacity:0.8}.mw-parser-output .mw-tpl-colorbox-title-corner{float:right;font-size:0.7em}.mw-parser-output .mw-tpl-colorbox-content{padding:0.5rem 1rem}.mw-parser-output .mw-tpl-colorbox-content::after{content:"";display:block;clear:both}</style><div class="mw-tpl-colorbox-title mw-tpl-colorbox-title--linked" style=""><a href="/wiki/Data_Platform" title="Data Platform"><strong>Data Platform documentation</strong></a> </div><div class="mw-tpl-colorbox-content"> <p>Documentation for users and administrators of Data Platform systems, including the Data Lake, analytics tools, Event Platform, Metrics Platform, and data pipelines. </p> </div> </div> <div class="mw-tpl-colorbox" style=""><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199608"><div class="mw-tpl-colorbox-title mw-tpl-colorbox-title--linked" style=""><a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering" class="extiw" title="mw:Data Platform Engineering"><strong>Team and project documentation</strong></a> </div><div class="mw-tpl-colorbox-content"> <p>List of teams in the Data Platform Engineering group, links to their documentation, and information about team processes, current projects, and roadmaps. </p> </div> </div> <div class="mw-tpl-colorbox" style=""><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199608"><div class="mw-tpl-colorbox-title mw-tpl-colorbox-title--linked" style=""><a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering/Intake_Process" class="extiw" title="mw:Data Platform Engineering/Intake Process"><strong>How to contact</strong></a> </div><div class="mw-tpl-colorbox-content"> <p>To contact us, please use the <a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering/Intake_Process" class="extiw" title="mw:Data Platform Engineering/Intake Process">intake process</a>. </p> <ul><li>See <a href="/wiki/Data_Platform_Engineering/Contact" title="Data Platform Engineering/Contact">the legacy Data Engineering team Contact page</a> (<a href="https://phabricator.wikimedia.org/T364572" class="extiw" title="phab:T364572">phab:T364572</a>) for other options.</li></ul> </div> </div> <div class="mw-tpl-colorbox" style=""><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199608"><div class="mw-tpl-colorbox-title" style=""><strong>Data Engineering team norms and processes</strong> </div><div class="mw-tpl-colorbox-content"> <ul><li><a href="/wiki/Data_Platform_Engineering/Ops_week" title="Data Platform Engineering/Ops week">Operational duty rotation</a></li> <li><a href="/wiki/Data_Platform_Engineering/Meeting_norms" title="Data Platform Engineering/Meeting norms">Team meeting norms</a></li> <li><a href="/wiki/Data_Platform_Engineering/Show_your_work" class="mw-redirect" title="Data Platform Engineering/Show your work">Show your work!</a></li> <li><a href="/wiki/Data_Platform_Engineering/Team_roles_and_expectations" title="Data Platform Engineering/Team roles and expectations">Team roles and expectations</a></li></ul> </div> </div> <div class="mw-tpl-colorbox" style=""><link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199608"><div class="mw-tpl-colorbox-title" style=""><strong>Team learning materials</strong> </div><div class="mw-tpl-colorbox-content"> <ul><li><a href="/wiki/Data_Platform_Engineering/Onboarding" title="Data Platform Engineering/Onboarding">Onboarding</a></li> <li><a href="/wiki/Data_Platform_Engineering/Learning_Materials" title="Data Platform Engineering/Learning Materials">Learning materials</a></li> <li><a href="https://www.mediawiki.org/wiki/Guidelines_for_a_healthy_code_review_culture" class="extiw" title="mw:Guidelines for a healthy code review culture">Guidelines for a healthy code review culture</a></li></ul> </div> </div></div> <p><span id="docs"></span> </p> </section><div class="mw-heading mw-heading2 ext-discussiontools-init-section section-heading" onclick="mfTempOpenSection(1)"><span class="indicator mf-icon mf-icon-expand mf-icon--small"></span><h2 id="About_the_Data_Platform_Engineering_docs" data-mw-thread-id="h-About_the_Data_Platform_Engineering_docs"><span data-mw-comment-start="" id="h-About_the_Data_Platform_Engineering_docs"></span>About the Data Platform Engineering docs<span data-mw-comment-end="h-About_the_Data_Platform_Engineering_docs"></span></h2></div><section class="mf-section-1 collapsible-block" id="mf-section-1"> <div class="mw-heading mw-heading3"><h3 id="Content_structure" data-mw-thread-id="h-Content_structure-About_the_Data_Platform_Engineering_docs"><span data-mw-comment-start="" id="h-Content_structure-About_the_Data_Platform_Engineering_docs"></span>Content structure<span data-mw-comment-end="h-Content_structure-About_the_Data_Platform_Engineering_docs"></span></h3></div> <table class="wikitable"> <caption>Caption text </caption> <tbody><tr> <th>Path</th> <th>Intended contents</th> <th>Category tag </th></tr> <tr> <td>Data_Platform/</td> <td>Documentation primarily meant for users of the Data Platform and its systems.</td> <td><a href="/wiki/Category:Data_platform" title="Category:Data platform">category:Data_platform</a> </td></tr> <tr> <td>Data_Platform/Systems</td> <td>Technical documentation focused on the administration and maintenance of the infrastructure, pipelines, components, and systems that make up the Data Platform. is under Category:Data_platform.</td> <td><a href="/wiki/Category:Data_platform" title="Category:Data platform">category:Data_platform</a> AND <a href="/wiki/Category:Data_platform_systems" title="Category:Data platform systems">category:Data_platform_systems</a> </td></tr> <tr> <td>Data_Platform_Engineering/</td> <td>Team pages left over from the migration of the former Data_Engineering docs. Their content covers procedures and team processes, so it should be integrated into the <a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering" class="extiw" title="mw:Data Platform Engineering">relevant team or organizational pages on mediawiki.org</a> (see <a href="https://phabricator.wikimedia.org/T367580" class="extiw" title="phab:T367580">phab:T367580</a> and <a href="https://phabricator.wikimedia.org/T364572" class="extiw" title="phab:T364572">phab:T364572</a>).</td> <td>varies </td></tr></tbody></table> <div class="mw-heading mw-heading3"><h3 id="Page_categories" data-mw-thread-id="h-Page_categories-About_the_Data_Platform_Engineering_docs"><span data-mw-comment-start="" id="h-Page_categories-About_the_Data_Platform_Engineering_docs"></span>Page categories<span data-mw-comment-end="h-Page_categories-About_the_Data_Platform_Engineering_docs"></span></h3></div> <p>Categories are easier to maintain and less disruptive to change than page structure. Categories also enable us to organize and navigate pages along multiple axes simultaneously, regardless of where the page is located in the content structure. Expand the section below to see the major categories that exist for these docs. </p><p>Disclaimer: this is an incomplete list! Other categories on Wikitech could be useful for some of these docs, and the docs may already be tagged in other categories not listed here. </p> <table class="mw-collapsible mw-collapsed" style="margin: 2px 0; background: #FFF; color: #000; width: 100%; border: 1px solid #AAA; padding: 0; border-spacing: 0; border-collapse: collapse;"> <tbody><tr> <th style="padding: 2px;"><div style="background:#CCF;padding:2px 0.5em;font-weight:bold;text-align:center;"> Category list </div> </th></tr> <tr> <td style="padding: 2px; background:transparent;font-weight:normal;text-align:left;"> <p>Categories for <b>types</b> of data platform systems/components: </p> <ul><li><a href="/wiki/Category:Data_pipelines" title="Category:Data pipelines">Data_pipelines</a></li> <li><a href="/wiki/Category:Data_stream" title="Category:Data stream">Data_stream</a></li> <li><a href="/wiki/Category:Query_engines" title="Category:Query engines">Query_engines</a></li></ul> <p>Categories for <b>specific</b> data platform systems/components: </p> <ul><li><a href="/wiki/Category:Analytics_cluster" title="Category:Analytics cluster">Analytics_cluster</a></li> <li><a href="/wiki/Category:Analytics_Query_Service" title="Category:Analytics Query Service">Analytics_Query_Service</a></li> <li><a href="/wiki/Category:Dashiki" title="Category:Dashiki">Dashiki</a></li> <li><a href="/wiki/Category:Dumps" title="Category:Dumps">Dumps</a></li> <li><a href="/wiki/Category:Spark" title="Category:Spark">Spark</a></li> <li><a href="/wiki/Category:Hive" title="Category:Hive">Hive</a></li> <li><a href="/wiki/Category:Presto" title="Category:Presto">Presto</a></li> <li><a href="/wiki/Category:WDQS" title="Category:WDQS">WDQS</a></li> <li><a href="/wiki/Category:Wikistats" title="Category:Wikistats">Wikistats</a></li></ul> <p>Categories for topics covered by the data platform documentation: </p> <ul><li><a href="/wiki/Category:Data_domains" title="Category:Data domains">Data_domains</a> <ul><li><a href="/wiki/Category:Contribution_data" title="Category:Contribution data">Contribution_data</a> <ul><li><a href="/wiki/Category:Edits_data" title="Category:Edits data">Edits_data</a></li> <li><a href="/wiki/Category:Editors_data" title="Category:Editors data">Editors_data</a></li></ul></li> <li><a href="/wiki/Category:Traffic_data" title="Category:Traffic data">Traffic_data</a> <ul><li><a href="/wiki/Category:Pageviews" title="Category:Pageviews">Pageviews</a></li> <li><a href="/wiki/Category:Unique_devices" title="Category:Unique devices">Unique_devices</a></li></ul></li> <li><a href="/wiki/Category:Content_data" title="Category:Content data">Content_data</a></li></ul></li> <li><a href="/wiki/Category:Metrics" title="Category:Metrics">Metrics</a></li></ul> <p>Categories for types of documentation (in order of their usage within the data platform docs, from most-used to least-used): </p> <ul><li><a href="/wiki/Category:Query_examples" title="Category:Query examples">Query_examples</a></li> <li><a href="/wiki/Category:Decision_log" title="Category:Decision log">Decision logs</a></li> <li><a href="/wiki/Category:Portals" title="Category:Portals">Portals</a></li> <li><a href="/wiki/Category:Landing_page" title="Category:Landing page">Landing pages</a></li> <li><a href="/wiki/Category:Tutorials" title="Category:Tutorials">Tutorials</a></li></ul> <p>Categories for maintaining / navigationg the docs (see details in the previous section): </p> <ul><li><a href="/wiki/Category:Data_platform" title="Category:Data platform">Data platform</a></li> <li><a href="/wiki/Category:Data_platform_systems" title="Category:Data platform systems">Data_platform_systems</a></li></ul> </td></tr></tbody></table> <div class="mw-heading mw-heading3"><h3 id="FAQs_and_guidelines_for_maintaining_these_docs" data-mw-thread-id="h-FAQs_and_guidelines_for_maintaining_these_docs-About_the_Data_Platform_Engineering_docs"><span data-mw-comment-start="" id="h-FAQs_and_guidelines_for_maintaining_these_docs-About_the_Data_Platform_Engineering_docs"></span>FAQs and guidelines for maintaining these docs<span data-mw-comment-end="h-FAQs_and_guidelines_for_maintaining_these_docs-About_the_Data_Platform_Engineering_docs"></span></h3></div> <style data-mw-deduplicate="TemplateStyles:r2211903">.mw-parser-output .note{background-position:left 7px top 50%;padding:0.5em 0.5em 0.5em 40px;margin:0.5em 0;overflow:hidden;background-color:#f8f9fa;color:#333;background-repeat:no-repeat;border:1px solid #ddd}.mw-parser-output .note-inline{display:inline-block;vertical-align:middle}.mw-parser-output .note-info{background-color:#eaf3ff;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg");background-size:25px;border-color:#a3caff;padding-left:40px;min-height:25px}.mw-parser-output .note-reminder{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/a/a8/OOjs_UI_icon_lightbulb-yellow.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-warn{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/3/3b/OOjs_UI_icon_alert-warning.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-error{background-color:#fee7e6;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/b/bf/OOjs_UI_icon_notice-destructive.svg");background-size:25px;border-color:#c33;min-height:25px}@media screen{html.skin-theme-clientpref-night .mw-parser-output .note{background-color:transparent;color:inherit}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .note{background-color:transparent;color:inherit}}</style><div role="note" class="note note-info"> Find techncial documentation guidance and templates at <a href="https://www.mediawiki.org/wiki/Documentation" class="extiw" title="mw:Documentation">mw:Documentation</a>.</div> <dl><dt>Where to put decision records?</dt> <dd>This may vary by product or project. The Data Platform Engineering teams already have multiple places where these docs may be living. Some are on Wikitech at <a href="/wiki/Metrics_Platform/Decision_Records" title="Metrics Platform/Decision Records">Metrics_Platform/Decision_Records</a>, or in pages under <a href="/wiki/Data_Platform/Evaluations" title="Data Platform/Evaluations">/Evaluations</a>. Some are on mediawiki.org at <a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering/Data_Products/Decision_Records" class="extiw" title="mw:Data Platform Engineering/Data Products/Decision Records">Data_Platform_Engineering/Data_Products/Decision_Records</a>. For systems or products that already have a decision record somewhere, it may be best to continue that pattern and keep things consistent, but you should link to your decision record location from other locations where people might look for it. If you publish on Wikitech, add <a href="/wiki/Category:Decision_log" title="Category:Decision log">Category:Decision_log</a>. Tip: the technical documentation toolkit has a <a href="https://www.mediawiki.org/wiki/Documentation/Decision_records" class="extiw" title="mw:Documentation/Decision records">Decision log template</a>.</dd></dl> <dl><dt>Where to put evaluations or design docs?</dt> <dd>This may vary by product or project. For systems or products that <a href="/wiki/Data_Platform/Evaluations" title="Data Platform/Evaluations">already have evaluations or design docs on Wikitech</a>, it may be simplest to continue that pattern. For those that don't have any extant evaluations or design docs, it's up to you. Consider your primary audience, and put the documentation in the place they're most likely to look for it. Then, add cross-references to and from the other places where people might look for the documentation. If you publish on Wikitech, add the</dd></dl> <dl><dt>Where to put project updates and product roadmaps?</dt> <dd>In Phabricator, and/or with the team's pages on mediawiki.org. For example: <a href="https://www.mediawiki.org/wiki/Data_Platform_Engineering/Data_Products/work_focus" class="extiw" title="mw:Data Platform Engineering/Data Products/work focus">Data_Products/work_focus</a>.</dd></dl> <dl><dt>Where to put metrics documentation?</dt> <dd>This may vary by product or project. Consider your primary audience for the metrics dataset, and put the documentation in the place they're most likely to look for it. Then, add cross-references to and from the other places where people might look for the documentation. So, for example, <a href="https://commons.wikimedia.org/wiki/Commons:WMF_support_for_Commons/Commons_Impact_Metrics" class="extiw" title="commons:Commons:WMF support for Commons/Commons Impact Metrics">Commons Impact Metrics documentation</a> (of the dataset, not the project) may be published under <a href="/wiki/Data_Platform/Data_Lake" title="Data Platform/Data Lake">Data_Platform/Data_Lake</a> so that it's collocated with many other dataset documentation pages. More important than where the docs live is that you add links to those docs in DataHub, from the project pages, and anywhere else someone may be looking when they're in need of that information.</dd></dl> <p>Keys to sustainable doc maintenance: </p> <ul><li>Apply categories to pages: this helps them remain discoverable through methods other than relying only on page structure / prefixing.</li> <li>Be consistent about where you put docs related to a given product or component. Then, add cross-references between that place and all the other places people might expect to find the information (other wikis, DataHub, Github READMEs, etc.).</li> <li>Avoid many levels of deep page nesting (more than 3 is probably too deep).</li> <li>Docs on Wikitech should generally be documentating how to use or administer a technology/system, not documenting things about the team or org that maintains a given technology/system. (Note: this is not a pattern that has consistently been followed in the past, so the current state of docs on Wikitech doesn't always reflect it.)</li></ul>    </section></div> <noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://wikitech.wikimedia.org/w/index.php?title=Data_Platform_Engineering&oldid=2204281">https://wikitech.wikimedia.org/w/index.php?title=Data_Platform_Engineering&oldid=2204281</a>"</div></div> </div> <div class="post-content" id="page-secondary-actions"> </div> </main> <footer class="mw-footer minerva-footer" role="contentinfo"> <a class="last-modified-bar" href="/w/index.php?title=Data_Platform_Engineering&action=history"> <div class="post-content last-modified-bar__content"> <span class="minerva-icon minerva-icon-size-medium minerva-icon--modified-history"></span> <span class="last-modified-bar__text modified-enhancement" data-user-name="Triciaburmeister" data-user-gender="female" data-timestamp="1720535110"> <span>Last edited on 9 July 2024, at 14:25</span> </span> <span class="minerva-icon minerva-icon-size-small minerva-icon--expand"></span> </div> </a> <div class="post-content footer-content"> <div id="p-lang"> <h4>Languages</h4> <section> <ul id="p-variants" class="minerva-languages"></ul> <p>This page is not available in other languages.</p> </section> </div> <div class="minerva-footer-logo"><img src="/static/images/mobile/copyright/wikitech-wordmark.svg" alt="Wikitech" width="140" height="27" style="width: 8.75em; height: 1.6875em;"/> </div> <ul id="footer-info" class="footer-info hlist hlist-separated"> <li id="footer-info-lastmod"> This page was last edited on 9 July 2024, at 14:25.</li> <li id="footer-info-copyright">Content is available under <a class="external" rel="nofollow" href="https://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a> unless otherwise noted.</li> </ul> <ul id="footer-places" class="footer-places hlist hlist-separated"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Main_Page">About Wikitech</a></li> <li id="footer-places-disclaimers"><a href="https://foundation.wikimedia.org/wiki/General_disclaimer">Disclaimers</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/wikitech.wikimedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-terms-use"><a href="https://foundation.m.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use">Terms of Use</a></li> <li id="footer-places-desktop-toggle"><a id="mw-mf-display-toggle" href="//wikitech.wikimedia.org/w/index.php?title=Data_Platform_Engineering&mobileaction=toggle_view_desktop" data-event-name="switch_to_desktop">Desktop</a></li> </ul> </div> </footer> </div> </div> <div class="mw-notification-area" data-mw="interface"></div>  <script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-4qzqr","wgBackendResponseTime":95,"wgDiscussionToolsPageThreads":[{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-About_the_Data_Platform_Engineering_docs","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Content_structure-About_the_Data_Platform_Engineering_docs","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Page_categories-About_the_Data_Platform_Engineering_docs","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-FAQs_and_guidelines_for_maintaining_these_docs-About_the_Data_Platform_Engineering_docs","replies":[]}]}],"wgPageParseReport":{"discussiontools":{"limitreport-timeusage":"0.012"},"limitreport":{"cputime":"0.063","walltime":"0.080","ppvisitednodes":{"value":299,"limit":1000000},"postexpandincludesize":{"value":7802,"limit":2097152},"templateargumentsize":{"value":5572,"limit":2097152},"expansiondepth":{"value":7,"limit":100},"expensivefunctioncount":{"value":0,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":7997,"limit":5000000},"timingprofile":["100.00% 46.213 1 -total"," 55.81% 25.790 1 Template:ContentGrid"," 34.55% 15.965 5 Template:Colored_box"," 25.66% 11.859 1 Template:Note"," 9.08% 4.198 1 Template:Anchor"," 8.83% 4.082 1 Template:Hidden"]},"cachereport":{"origin":"mw-web.eqiad.main-5dc468848-4mq4l","timestamp":"20241123191814","ttl":2592000,"transientcontent":false}}});});</script> <script>(window.NORLQ=window.NORLQ||[]).push(function(){var ns,i,p,img;ns=document.getElementsByTagName('noscript');for(i=0;i<ns.length;i++){p=ns[i].nextSibling;if(p&&p.className&&p.className.indexOf('lazy-image-placeholder')>-1){img=document.createElement('img');img.setAttribute('src',p.getAttribute('data-src'));img.setAttribute('width',p.getAttribute('data-width'));img.setAttribute('height',p.getAttribute('data-height'));img.setAttribute('alt',p.getAttribute('data-alt'));p.parentNode.replaceChild(img,p);}}});</script> </body> </html>