CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Portal:Toolforge/Admin - Wikitech</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )labswikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":true,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat": "dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"fa0dc45b-d767-4146-b600-f3b0a82a2a3b","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Portal:Toolforge/Admin","wgTitle":"Portal:Toolforge/Admin","wgCurRevisionId":2241510,"wgRevisionId":2241510,"wgArticleId":21478,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Toolforge admin","Toolforge","Cloud Services admin"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Portal:Toolforge/Admin","wgRelevantArticleId":21478,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikitech","wgCiteReferencePreviewsActive":true,"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true, "wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":20000,"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgDiscussionToolsFeaturesEnabled":{"replytool":true,"newtopictool":true,"sourcemodetoolbar":true,"topicsubscription":false,"autotopicsub":false,"visualenhancements":false,"visualenhancements_reply":false,"visualenhancements_pageframe":false},"wgDiscussionToolsFallbackEditMode":"visual","wgULSPosition":"personal","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"wgSiteNoticeId":"2.0"};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles": "ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.inputBox.styles":"ready","ext.pygments":"ready","mediawiki.special":"ready","oojs-ui-core.styles":"ready","oojs-ui.styles.indicators":"ready","mediawiki.widgets.styles":"ready","oojs-ui-core.icons":"ready","mediawiki.htmlform.ooui.styles":"ready","ext.discussionTools.init.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.pt":"ready","ext.dismissableSiteNotice.styles":"ready"};RLPAGEMODULES=["ext.pygments.view","mediawiki.page.media","site","mediawiki.page.ready","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.site","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader", "ext.echo.centralauth","ext.discussionTools.init","ext.eventLogging","ext.wikimediaEvents","ext.uls.interface","ext.checkUser.clientHints","ext.dismissableSiteNotice"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.discussionTools.init.styles%7Cext.dismissableSiteNotice.styles%7Cext.inputBox.styles%7Cext.pygments%7Cext.uls.pt%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cmediawiki.htmlform.ooui.styles%7Cmediawiki.special%7Cmediawiki.widgets.styles%7Coojs-ui-core.icons%2Cstyles%7Coojs-ui.styles.indicators%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Portal:Toolforge/Admin - Wikitech"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="icon" href="/static/favicon/wikitech.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikitech (en)"> <link rel="EditURI" type="application/rsd+xml" href="//wikitech.wikimedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://wikitech.wikimedia.org/wiki/Portal:Toolforge/Admin"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/"> <link rel="alternate" type="application/atom+xml" title="Wikitech Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="ext-discussiontools-replytool-enabled ext-discussiontools-newtopictool-enabled ext-discussiontools-sourcemodetoolbar-enabled skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Portal_Toolforge_Admin rootpage-Portal_Toolforge skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-Server-admin-log:-Prod" class="mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Server admin log: Prod</span></a></li><li id="n-Admin-log:-RelEng" class="mw-list-item"><a href="/wiki/Release_Engineering/SAL"><span>Admin log: RelEng</span></a></li><li id="n-Incident-status" class="mw-list-item"><a href="/wiki/Incident_status"><span>Incident status</span></a></li><li id="n-Deployments" class="mw-list-item"><a href="/wiki/Deployments"><span>Deployments</span></a></li><li id="n-SRE-Team-Help" class="mw-list-item"><a href="/wiki/SRE/SRE_Team_requests"><span>SRE Team Help</span></a></li> </ul> </div> </div> <div id="p-Cloud_VPS_&_Toolforge" class="vector-menu mw-portlet mw-portlet-Cloud_VPS_Toolforge" > <div class="vector-menu-heading"> Cloud VPS & Toolforge </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-Cloud-VPS-portal" class="mw-list-item"><a href="/wiki/Portal:Cloud_VPS"><span>Cloud VPS portal</span></a></li><li id="n-Toolforge-portal" class="mw-list-item"><a href="/wiki/Portal:Toolforge"><span>Toolforge portal</span></a></li><li id="n-Request-VPS-project" class="mw-list-item"><a href="https://phabricator.wikimedia.org/project/view/2875/"><span>Request VPS project</span></a></li><li id="n-Admin-log:-Cloud-VPS" class="mw-list-item"><a href="/wiki/Cloud_VPS_Server_Admin_Log"><span>Admin log: Cloud VPS</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikitech.svg" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikitech" src="/static/images/mobile/copyright/wikitech-wordmark.svg" style="width: 8.75em; height: 1.6875em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikitech [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikitech" aria-label="Search Wikitech" autocapitalize="sentences" title="Search Wikitech [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-uls" class="mw-list-item active user-links-collapsible-item"><a data-mw="interface" href="#" class="uls-trigger cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>English</span></a> </li> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?utm_source=donate&utm_medium=sidebar&utm_campaign=spontaneous&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Portal%3AToolforge%2FAdmin" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out user-links-collapsible-item" title="More options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?utm_source=donate&utm_medium=sidebar&utm_campaign=spontaneous&uselang=en"><span>Donate</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Portal%3AToolforge%2FAdmin" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"><div id="mw-dismissablenotice-anonplace"></div><script>(function(){var node=document.getElementById("mw-dismissablenotice-anonplace");if(node){node.outerHTML="\u003Cdiv class=\"mw-dismissable-notice\"\u003E\u003Cdiv class=\"mw-dismissable-notice-close\"\u003E[\u003Ca tabindex=\"0\" role=\"button\"\u003Edismiss\u003C/a\u003E]\u003C/div\u003E\u003Cdiv class=\"mw-dismissable-notice-body\"\u003E\u003C!-- CentralNotice --\u003E\u003Cdiv id=\"localNotice\" data-nosnippet=\"\"\u003E\u003Cdiv class=\"sitenotice\" lang=\"en\" dir=\"ltr\"\u003E\u003Ctable style=\"width: 75%; background-color: var(--background-color-warning-subtle, #fdf2d5); border: var(--border-subtle, 1px solid #987027); color: var(--color-base, #202122); border-radius: 10px; padding: 5px; margin: 0 auto;\"\u003E\n\u003Ctbody\u003E\u003Ctr\u003E\n\u003Ctd style=\"width:40px; height:40px; text-align:center; vertical-align:middle; padding: 2px;\"\u003E\u003Cspan typeof=\"mw:File\"\u003E\u003Ca href=\"/wiki/File:OOjs_UI_icon_alert-warning.svg\" class=\"mw-file-description\"\u003E\u003Cimg src=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/30px-OOjs_UI_icon_alert-warning.svg.png\" decoding=\"async\" width=\"30\" height=\"30\" class=\"mw-file-element\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/45px-OOjs_UI_icon_alert-warning.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/60px-OOjs_UI_icon_alert-warning.svg.png 2x\" data-file-width=\"20\" data-file-height=\"20\" /\u003E\u003C/a\u003E\u003C/span\u003E\n\u003C/td\u003E\n\u003Ctd style=\"text-align:center; vertical-align: middle; padding: 4px; max-height: 60px;\"\u003E\u003Cb\u003EWe are migrating Wikitech to \u003Ca href=\"/wiki/Wikitech/SUL-migration\" title=\"Wikitech/SUL-migration\"\u003ESUL\u003C/a\u003E!\u003C/b\u003E\n\u003Cp\u003E\u003Cb\u003EAction may be required for your \u003Ca href=\"/wiki/Wikitech/SUL-migration#What_You_Should_Do\" title=\"Wikitech/SUL-migration\"\u003E account\u003C/a\u003E!\u003C/b\u003E\n\u003C/p\u003E\u003Cp\u003E\u003Cb\u003ETrouble logging in? Please visit \u003Ca href=\"https://phabricator.wikimedia.org/T376267\" class=\"extiw\" title=\"phab:T376267\"\u003ET376267\u003C/a\u003E\u003C/b\u003E\n\u003C/p\u003E\n\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E";}}());</script></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">Beginning</div> </a> </li> <li id="toc-Admin_permissions" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Admin_permissions"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Admin permissions</span> </div> </a> <ul id="toc-Admin_permissions-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Failover" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Failover"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Failover</span> </div> </a> <button aria-controls="toc-Failover-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Failover subsection</span> </button> <ul id="toc-Failover-sublist" class="vector-toc-list"> <li id="toc-WebProxy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#WebProxy"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1</span> <span>WebProxy</span> </div> </a> <ul id="toc-WebProxy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Static_webserver" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Static_webserver"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.2</span> <span>Static webserver</span> </div> </a> <ul id="toc-Static_webserver-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Checker_service" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Checker_service"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.3</span> <span>Checker service</span> </div> </a> <ul id="toc-Checker_service-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Redis" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Redis"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.4</span> <span>Redis</span> </div> </a> <ul id="toc-Redis-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Prometheus" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Prometheus"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.5</span> <span>Prometheus</span> </div> </a> <ul id="toc-Prometheus-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Services" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Services"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.6</span> <span>Services</span> </div> </a> <ul id="toc-Services-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Command_orchestration" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Command_orchestration"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Command orchestration</span> </div> </a> <ul id="toc-Command_orchestration-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Administrative_tasks" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Administrative_tasks"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Administrative tasks</span> </div> </a> <button aria-controls="toc-Administrative_tasks-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Administrative tasks subsection</span> </button> <ul id="toc-Administrative_tasks-sublist" class="vector-toc-list"> <li id="toc-Logging_in_as_root" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Logging_in_as_root"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.1</span> <span>Logging in as root</span> </div> </a> <ul id="toc-Logging_in_as_root-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Disabling_all_ssh_logins_except_root" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Disabling_all_ssh_logins_except_root"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.2</span> <span>Disabling all ssh logins except root</span> </div> </a> <ul id="toc-Disabling_all_ssh_logins_except_root-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Complaints_of_bastion_being_slow" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Complaints_of_bastion_being_slow"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.3</span> <span>Complaints of bastion being slow</span> </div> </a> <ul id="toc-Complaints_of_bastion_being_slow-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Local_package_management" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Local_package_management"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4</span> <span>Local package management</span> </div> </a> <ul id="toc-Local_package_management-sublist" class="vector-toc-list"> <li id="toc-Local_package_policy" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Local_package_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.1</span> <span>Local package policy</span> </div> </a> <ul id="toc-Local_package_policy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Building_packages" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Building_packages"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.2</span> <span>Building packages</span> </div> </a> <ul id="toc-Building_packages-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deploy_new_misctools_package" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Deploy_new_misctools_package"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.3</span> <span>Deploy new misctools package</span> </div> </a> <ul id="toc-Deploy_new_misctools_package-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Testing/QA_for_a_new_tools-webservice_package" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Testing/QA_for_a_new_tools-webservice_package"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.4</span> <span>Testing/QA for a new tools-webservice package</span> </div> </a> <ul id="toc-Testing/QA_for_a_new_tools-webservice_package-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deploy_new_tools-webservice_package" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Deploy_new_tools-webservice_package"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.5</span> <span>Deploy new tools-webservice package</span> </div> </a> <ul id="toc-Deploy_new_tools-webservice_package-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Webserver_statistics" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Webserver_statistics"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.5</span> <span>Webserver statistics</span> </div> </a> <ul id="toc-Webserver_statistics-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Banning_an_IP_from_tool_labs" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Banning_an_IP_from_tool_labs"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.6</span> <span>Banning an IP from tool labs</span> </div> </a> <ul id="toc-Banning_an_IP_from_tool_labs-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deploying_the_main_web_page" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Deploying_the_main_web_page"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.7</span> <span>Deploying the main web page</span> </div> </a> <ul id="toc-Deploying_the_main_web_page-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Regenerate_replica.my.cnf" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Regenerate_replica.my.cnf"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.8</span> <span>Regenerate replica.my.cnf</span> </div> </a> <ul id="toc-Regenerate_replica.my.cnf-sublist" class="vector-toc-list"> <li id="toc-Debugging_bad_MariaDB_credentials" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Debugging_bad_MariaDB_credentials"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.8.1</span> <span>Debugging bad MariaDB credentials</span> </div> </a> <ul id="toc-Debugging_bad_MariaDB_credentials-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Regenerate_kubernetes_credentials_for_tools_(.kube/config)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Regenerate_kubernetes_credentials_for_tools_(.kube/config)"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.9</span> <span>Regenerate kubernetes credentials for tools (.kube/config)</span> </div> </a> <ul id="toc-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Adding_K8S_Components" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Adding_K8S_Components"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.10</span> <span>Adding K8S Components</span> </div> </a> <ul id="toc-Adding_K8S_Components-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deleting_a_tool" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Deleting_a_tool"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.11</span> <span>Deleting a tool</span> </div> </a> <ul id="toc-Deleting_a_tool-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-SSL_certificates" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#SSL_certificates"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.12</span> <span>SSL certificates</span> </div> </a> <ul id="toc-SSL_certificates-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Granting_a_tool_write_access_to_Elasticsearch" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Granting_a_tool_write_access_to_Elasticsearch"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.13</span> <span>Granting a tool write access to Elasticsearch</span> </div> </a> <ul id="toc-Granting_a_tool_write_access_to_Elasticsearch-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Package_upgrades" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Package_upgrades"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.14</span> <span>Package upgrades</span> </div> </a> <ul id="toc-Package_upgrades-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.15</span> <span>Creating a new Docker image (e.g. for new versions of Node.js)</span> </div> </a> <ul id="toc-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Kubernetes" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Kubernetes"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Kubernetes</span> </div> </a> <ul id="toc-Kubernetes-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Build_service" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Build_service"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>Build service</span> </div> </a> <ul id="toc-Build_service-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Tools-mail_/_Exim" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Tools-mail_/_Exim"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>Tools-mail / Exim</span> </div> </a> <ul id="toc-Tools-mail_/_Exim-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Users_and_community" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Users_and_community"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>Users and community</span> </div> </a> <button aria-controls="toc-Users_and_community-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Users and community subsection</span> </button> <ul id="toc-Users_and_community-sublist" class="vector-toc-list"> <li id="toc-Project_membership_request_approval" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Project_membership_request_approval"> <div class="vector-toc-text"> <span class="vector-toc-numb">8.1</span> <span>Project membership request approval</span> </div> </a> <ul id="toc-Project_membership_request_approval-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Quota_management" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Quota_management"> <div class="vector-toc-text"> <span class="vector-toc-numb">8.2</span> <span>Quota management</span> </div> </a> <ul id="toc-Quota_management-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Other" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Other"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>Other</span> </div> </a> <button aria-controls="toc-Other-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Other subsection</span> </button> <ul id="toc-Other-sublist" class="vector-toc-list"> <li id="toc-How_do_Toolforge_web_services_actually_work?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#How_do_Toolforge_web_services_actually_work?"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.1</span> <span>How do Toolforge web services actually work?</span> </div> </a> <ul id="toc-How_do_Toolforge_web_services_actually_work?-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-What_makes_a_root/Giving_root_access" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#What_makes_a_root/Giving_root_access"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.2</span> <span>What makes a root/Giving root access</span> </div> </a> <ul id="toc-What_makes_a_root/Giving_root_access-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Servicegroup_log" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Servicegroup_log"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.3</span> <span>Servicegroup log</span> </div> </a> <ul id="toc-Servicegroup_log-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Useful_administrative_tools" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Useful_administrative_tools"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.4</span> <span>Useful administrative tools</span> </div> </a> <ul id="toc-Useful_administrative_tools-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Brainstorming" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Brainstorming"> <div class="vector-toc-text"> <span class="vector-toc-numb">10</span> <span>Brainstorming</span> </div> </a> <ul id="toc-Brainstorming-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Sub_pages" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Sub_pages"> <div class="vector-toc-text"> <span class="vector-toc-numb">11</span> <span>Sub pages</span> </div> </a> <ul id="toc-Sub_pages-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Portal:Toolforge/Admin</span></h1> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Portal:Toolforge/Admin" title="View the content page [c]" accesskey="c"><span>Page</span></a></li><li id="ca-talk" class="new vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Talk:Portal:Toolforge/Admin&action=edit&redlink=1" rel="discussion" class="new" title="Discussion about the content page (page does not exist) [t]" accesskey="t"><span>Discussion</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Portal:Toolforge/Admin"><span>Read</span></a></li><li id="ca-viewsource" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=edit" title="This page is protected.
You can view its source [e]" accesskey="e"><span>View source</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Portal:Toolforge/Admin"><span>Read</span></a></li><li id="ca-more-viewsource" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=edit"><span>View source</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Portal:Toolforge/Admin" title="A list of all wiki pages that link here [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Portal:Toolforge/Admin" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&oldid=2241510" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Portal%3AToolforge%2FAdmin&id=2241510&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FPortal%3AToolforge%2FAdmin"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FPortal%3AToolforge%2FAdmin"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-create_a_book" class="mw-list-item"><a href="/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Portal%3AToolforge%2FAdmin"><span>Create a book</span></a></li><li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Portal%3AToolforge%2FAdmin&action=show-download-screen"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikitech</div> </div> <div id="contentSub"><div id="mw-content-subtitle"><div class="subpages">< <bdi dir="ltr"><a href="/wiki/Portal:Toolforge" title="Portal:Toolforge">Portal:Toolforge</a></bdi></div></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><style data-mw-deduplicate="TemplateStyles:r2241375">.mw-parser-output .tpl-navsidebar{max-width:22em;background:var(--background-color-base,#fff);color:var(--color-base,#202122);border:1px solid var(--border-color-base,#a2a9b1);float:right;clear:right;margin:.5em 0 1em 1em}.mw-parser-output .tpl-navsidebar-floatright{float:right;clear:right;margin:.5em 0 1em 1em}.mw-parser-output .tpl-navsidebar-floatleft{float:left;clear:left;margin:.5em 1em 1em 0}.mw-parser-output .tpl-navsidebar-floatnone{float:none;clear:both;margin:.5em 0}.mw-parser-output .tpl-navsidebar-topimage{margin:0 0 16px 0}.mw-parser-output .tpl-navsidebar-title{margin:8px 16px;border-bottom:3px solid var(--border-color-muted,#eaecf0);font-size:20px;text-align:center}.mw-parser-output .tpl-navsidebar-image{margin:0 0 8px}.mw-parser-output .tpl-navsidebar-content{margin:0 0 16px 0;padding:0 8px}.mw-parser-output .tpl-navsidebar-heading{margin:8px 0;font-weight:bold}.mw-parser-output .tpl-navsidebar-foot{padding:0 8px;margin:0;text-align:right;font-size:smaller}@media not (min-width:720px){.mw-parser-output .tpl-navsidebar{float:none;clear:both;margin:.5em 0;max-width:none}}</style><div role="navigation" class="navigation-not-searchable tpl-navsidebar" style=""><p class="tpl-navsidebar-title"><a class="mw-selflink selflink">Toolforge Admin</a></p><div class="tpl-navsidebar-image"><figure class="mw-halign-center" typeof="mw:File"><a href="/wiki/File:Toolforge_logo.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Toolforge_logo.svg/50px-Toolforge_logo.svg.png" decoding="async" width="50" height="50" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Toolforge_logo.svg/75px-Toolforge_logo.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Toolforge_logo.svg/100px-Toolforge_logo.svg.png 2x" data-file-width="180" data-file-height="180"/></a><figcaption></figcaption></figure></div><div class="tpl-navsidebar-contents"><div class="tpl-navsidebar-content"> <div style="font-size: 85%;"> <div class="mw-inputbox-centered" style=""><form name="searchbox" class="searchbox mw-inputbox-form-inline" action="/wiki/Special:Search"><div class="cdx-text-input"><input class="mw-searchInput searchboxInput cdx-text-input__input" name="search" placeholder="Search Toolforge Admin Documentation" size="35" dir="ltr"/></div><input type="hidden" value=""Portal:Tool"" name="prefix"/> <input type="submit" name="fulltext" value="Search" class="cdx-button"/><input type="hidden" value="Search" name="fulltext"/></form></div> </div> </div><div class="tpl-navsidebar-content"> <ul><li><a href="/wiki/Help:Toolforge" title="Help:Toolforge">Toolforge user docs</a></li> <li><a class="mw-selflink selflink">Toolforge admin docs</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/System_Overview" title="Portal:Toolforge/Admin/System Overview">System overview</a></li> <li><a href="/wiki/Wikimedia_Cloud_Services_team/Ownership" title="Wikimedia Cloud Services team/Ownership">Reference: all WMCS-owned services</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Admin tasks and tools</p><p class="mw-empty-elt"> </p><ul><li><a class="mw-selflink-fragment" href="#Administrative_tasks">List of admin tasks</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a></li> <li><a class="external text" href="https://tool-db-usage.toolforge.org/">ToolsDB</a></li> <li><a class="external text" href="https://openstack-browser.toolforge.org/">OpenStack Browser</a></li> <li><a class="external text" href="https://toolhub.wikimedia.org/lists/192">Infrastructure tools</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading"><a class="mw-selflink-fragment" href="#Failover">Failover</a></p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Toolschecker" title="Portal:Toolforge/Admin/Toolschecker"> Toolschecker</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Redis" title="Portal:Toolforge/Admin/Redis"> Redis</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Prometheus#Failover" title="Portal:Toolforge/Admin/Prometheus"> Prometheus</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading"><a href="/wiki/Portal:Toolforge/Admin/Kubernetes" title="Portal:Toolforge/Admin/Kubernetes">Kubernetes</a></p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Certificates" title="Portal:Toolforge/Admin/Kubernetes/Certificates">Certificates</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Deploying" class="mw-redirect" title="Portal:Toolforge/Admin/Kubernetes/Deploying">Deploying</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Docker-registry" title="Portal:Toolforge/Admin/Kubernetes/Docker-registry">Docker-registry</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Jobs_framework" class="mw-redirect" title="Portal:Toolforge/Admin/Kubernetes/Jobs framework">Jobs framework</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Labels" title="Portal:Toolforge/Admin/Kubernetes/Labels">Labels</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Networking_and_ingress" title="Portal:Toolforge/Admin/Kubernetes/Networking and ingress">Networking and ingress</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Pod_tracing" title="Portal:Toolforge/Admin/Kubernetes/Pod tracing">Pod tracing</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Kubernetes services</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Jobs_framework" title="Portal:Toolforge/Admin/Jobs framework">Jobs framework</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Build_Service" title="Portal:Toolforge/Admin/Build Service">Build service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Envvars_Service" title="Portal:Toolforge/Admin/Envvars Service">Envvars service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Harbor" title="Portal:Toolforge/Admin/Harbor">Harbor</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Packaging</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">.deb packages</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Runbooks and emergency guides</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Runbooks" title="Portal:Toolforge/Admin/Runbooks">Portal:Toolforge/Admin/Runbooks</a></li> <li><a href="/wiki/Wikimedia_Cloud_Services_team/Incident_Response_Process" class="mw-redirect" title="Wikimedia Cloud Services team/Incident Response Process"> Incident response process</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Other</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Exim" title="Portal:Toolforge/Admin/Exim"> Exim</a> and <a href="/wiki/Portal:Cloud_VPS/Admin/Email#Operations" title="Portal:Cloud VPS/Admin/Email"> Email operations</a></li></ul> </div></div><p class="tpl-navsidebar-foot">[<span class="noprint plainlinks"><a class="external text" href="https://wikitech.wikimedia.org/w/index.php?title=Template:Toolforge_nav_admin&action=edit"><span title="Edit this template">edit</span></a></span>]</p></div> <p>Documentation of backend components and <b>admin procedures for Toolforge</b>. See <a href="/wiki/Help:Toolforge" title="Help:Toolforge">Help:Toolforge</a> for user facing documentation about actually using Toolforge to run your bots and webservices. </p> <meta property="mw:PageProp/toc"/> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Admin_permissions" data-mw-thread-id="h-Admin_permissions"><span data-mw-comment-start="" id="h-Admin_permissions"></span>Admin permissions<span data-mw-comment-end="h-Admin_permissions"></span></h2></div> <p>Performing admin procedures requires having admin permissions on Toolforge. There is not a single "admin" flag, but a set of interrelated permissions you can be granted. These are described in detail in the page <a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a>. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Failover" data-mw-thread-id="h-Failover"><span data-mw-comment-start="" id="h-Failover"></span>Failover<span data-mw-comment-end="h-Failover"></span></h2></div> <p>Tools should be able to survive the failure of any one virt* node. Some items may need manual failover </p> <div class="mw-heading mw-heading3"><h3 id="WebProxy" data-mw-thread-id="h-WebProxy-Failover"><span data-mw-comment-start="" id="h-WebProxy-Failover"></span>WebProxy<span data-mw-comment-end="h-WebProxy-Failover"></span></h3></div> <style data-mw-deduplicate="TemplateStyles:r2199615">.mw-parser-output .tracked{border:1px solid var(--border-color-base,#a2a9b1);border-radius:.5em;background-color:var(--background-color-interactive,#eaecf0);background-image:linear-gradient(var(--background-color-interactive,#eaecf0),var(--background-color-interactive-subtle,#f8f9fa));color:var(--color-base,#202122);font-size:85%;text-align:center;margin-bottom:1em;width:12em;padding:0.5em}.mw-parser-output .tracked-status{color:var(--color-emphasized,#000);font-weight:bold;text-transform:uppercase}.mw-parser-output .tracked-status--resolved{color:var(--color-success,#14866d)}.mw-parser-output .tracked-status--critical{color:var(--color-error,#d73333);font-size:1.5em}</style> <div style="float:right; clear:right; margin-left:1em;" class="plainlinks tracked mw-trackedTemplate">Tracked in <a href="https://phabricator.wikimedia.org/" class="extiw" title="phabricator:">Phabricator</a><br/><b><a href="https://phabricator.wikimedia.org/T283948" class="extiw" title="phabricator:T283948"><span class="trakfab-T283948">Task T283948</span></a></b></div> <p>The front web proxy is now a stateless web service. In case one dies, swap the floating IP to the other instance. </p> <div class="mw-heading mw-heading3"><h3 id="Static_webserver" data-mw-thread-id="h-Static_webserver-Failover"><span data-mw-comment-start="" id="h-Static_webserver-Failover"></span>Static webserver<span data-mw-comment-end="h-Static_webserver-Failover"></span></h3></div> <p>This is a stateless simple nginx http server. Simply switch the floating IP from tools-static-10 to tools-static-11 (or vice versa) to switch over. Recovery is also equally trivial - just bring the machine back up and make sure puppet is ok. </p> <div class="mw-heading mw-heading3"><h3 id="Checker_service" data-mw-thread-id="h-Checker_service-Failover"><span data-mw-comment-start="" id="h-Checker_service-Failover"></span>Checker service<span data-mw-comment-end="h-Checker_service-Failover"></span></h3></div> <p>This is the service that catchpoint (our external monitoring service) hits to check status of several services. It's totally stateless, so just switching the public IP from tools-checker-03 to -04 (or vice versa) should be fine. Same procedure as static webserver. </p><p>See <a href="/wiki/Portal:Toolforge/Admin/Toolschecker" title="Portal:Toolforge/Admin/Toolschecker">Portal:Toolforge/Admin/Toolschecker</a> </p> <div class="mw-heading mw-heading3"><h3 id="Redis" data-mw-thread-id="h-Redis-Failover"><span data-mw-comment-start="" id="h-Redis-Failover"></span>Redis<span data-mw-comment-end="h-Redis-Failover"></span></h3></div> <p>Redis uses <a href="/wiki/Portal:Toolforge/Admin/Redis" title="Portal:Toolforge/Admin/Redis">Sentinel</a> to automatically fail over in case of a node failure. </p> <div class="mw-heading mw-heading3"><h3 id="Prometheus" data-mw-thread-id="h-Prometheus-Failover"><span data-mw-comment-start="" id="h-Prometheus-Failover"></span>Prometheus<span data-mw-comment-end="h-Prometheus-Failover"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Prometheus#Failover" title="Portal:Toolforge/Admin/Prometheus">Portal:Toolforge/Admin/Prometheus#Failover</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Services" data-mw-thread-id="h-Services-Failover"><span data-mw-comment-start="" id="h-Services-Failover"></span>Services<span data-mw-comment-end="h-Services-Failover"></span></h3></div> <p>Service nodes run the Toolforge internal <b>aptly</b> service, to serve .deb packages as a repository for all the other nodes. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Command_orchestration" data-mw-thread-id="h-Command_orchestration"><span data-mw-comment-start="" id="h-Command_orchestration"></span>Command orchestration<span data-mw-comment-end="h-Command_orchestration"></span></h2></div> <p>Toolforge and Toolsbeta both have a local <a href="/wiki/Cumin" title="Cumin">cumin</a> server. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Administrative_tasks" data-mw-thread-id="h-Administrative_tasks"><span data-mw-comment-start="" id="h-Administrative_tasks"></span>Administrative tasks<span data-mw-comment-end="h-Administrative_tasks"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="Logging_in_as_root" data-mw-thread-id="h-Logging_in_as_root-Administrative_tasks"><span data-mw-comment-start="" id="h-Logging_in_as_root-Administrative_tasks"></span>Logging in as root<span data-mw-comment-end="h-Logging_in_as_root-Administrative_tasks"></span></h3></div> <p>For normal login root access see <a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a>. </p><p>In case the normal login does not work for example due to an LDAP failure, administrators can also directly log in as root. To prepare for that occasion, generate a separate key with <code>ssh-keygen</code>, add an entry to the <code>passwords::root::extra_keys</code> hash in Horizon's 'Project Puppet' section with your shell username as key and your public key as value and wait a Puppet cycle to have your key added to the <code>root</code> accounts. Add to your <code>~/.ssh/config</code>: </p> <pre># Use different identity for Tools root. Match host *.tools.eqiad1.wikimedia.cloud user root IdentityFile ~/.ssh/your_secret_root_key </pre> <p>The code that reads <code>passwords::root::extra_keys</code> is in <a class="external text" href="https://phabricator.wikimedia.org/diffusion/LPRI/browse/master/modules/passwords/manifests/init.pp">labs/private:modules/passwords/manifests/init.pp</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Disabling_all_ssh_logins_except_root" data-mw-thread-id="h-Disabling_all_ssh_logins_except_root-Administrative_tasks"><span data-mw-comment-start="" id="h-Disabling_all_ssh_logins_except_root-Administrative_tasks"></span>Disabling all ssh logins except root<span data-mw-comment-end="h-Disabling_all_ssh_logins_except_root-Administrative_tasks"></span></h3></div> <p>Useful for dealing with security critical situations. Just touch <code>/etc/nologin</code> and PAM will prevent any and all non-root logins. </p> <div class="mw-heading mw-heading3"><h3 id="Complaints_of_bastion_being_slow" data-mw-thread-id="h-Complaints_of_bastion_being_slow-Administrative_tasks"><span data-mw-comment-start="" id="h-Complaints_of_bastion_being_slow-Administrative_tasks"></span>Complaints of bastion being slow<span data-mw-comment-end="h-Complaints_of_bastion_being_slow-Administrative_tasks"></span></h3></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199615"> <div style="float:right; clear:right; margin-left:1em;" class="plainlinks tracked mw-trackedTemplate">Tracked in <a href="https://phabricator.wikimedia.org/" class="extiw" title="phabricator:">Phabricator</a><br/><b><a href="https://phabricator.wikimedia.org/T266300" class="extiw" title="phabricator:T266300"><span class="trakfab-T266300">Task T266300</span></a></b></div> <p>Users are increasingly noticing slowness on tools-login due to either CPU or IOPS exhaustion caused by people running processes there instead of on Kubernetes. Here are some tips for finding the processes in need of killing: </p> <ul><li>Look for IOPS hogs <ul><li><code class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><span class="gp">$ </span>iotop</code></li></ul></li> <li>Look for abnormal processes: <ul><li><code class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><span class="gp">$ </span>ps<span class="w"> </span>axo<span class="w"> </span>user:32,pid,cmd<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-Ev<span class="w"> </span><span class="s2">"^(</span><span class="nv">$USER</span><span class="s2">|root|daemon|_lldpd|messagebus|nagios|nslcd|ntp|prometheus|statd|syslog|Debian-exim|www-data)"</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-ivE<span class="w"> </span><span class="s1">'screen|tmux|-bash|mosh-server|sshd:|/bin/bash|/bin/zsh'</span></code></li> <li>If you see <code>pyb.py</code> kill with extreme prejudice.</li></ul></li> <li>If the rogue job is running as a tool, <code>!log</code> something like: <code>!log tools.$TOOL Killed $PROC process running on tools-bastion-NN. See https://wikitech.wikimedia.org/wiki/Help:Toolforge/Jobs_framework for instructions on running jobs on Kubernetes.</code></li></ul> <div class="mw-heading mw-heading3"><h3 id="Local_package_management" data-mw-thread-id="h-Local_package_management-Administrative_tasks"><span data-mw-comment-start="" id="h-Local_package_management-Administrative_tasks"></span>Local package management<span data-mw-comment-end="h-Local_package_management-Administrative_tasks"></span></h3></div> <p>Local packages are provided by an <code>aptly</code> repository on <code>tools-services-05</code>. </p><p>On <code>tools-services-05</code>, you can manipulate the package database by various commands; cf. <code>aptly(1)</code>. Afterwards, you need to publish the database to the file <code>Packages</code> by (for the <code>trusty-tools</code> repository) <code>aptly publish --skip-signing update trusty-tools</code>. To use the packages on the clients you need to wait 30 minutes again or run <code>apt-get update</code>. In general, you should never just delete packages, but move them to <code>~tools.admin/archived-packages</code>. </p><p>You can always see where a package is (would be) coming from with <code>apt-cache showpkg $package</code>. </p> <div class="mw-heading mw-heading4"><h4 id="Local_package_policy" data-mw-thread-id="h-Local_package_policy-Local_package_management"><span data-mw-comment-start="" id="h-Local_package_policy-Local_package_management"></span>Local package policy<span data-mw-comment-end="h-Local_package_policy-Local_package_management"></span></h4></div> <p><b>Package repositories</b> </p> <ul><li>We only install packages from trustworthy repositories. <ul><li>OK are <ul><li>The official Debian and Ubuntu repositories, and</li> <li>Self-built packages (apt.wikimedia.org and aptly)</li></ul></li> <li>Not OK are: <ul><li>PPAs</li> <li>other 3rd party repositories</li></ul></li></ul></li></ul> <p><i>Packagers effectively get root on our systems, as they could add a rootkit to the package, or upload an unsafe sshd version, and apt-get will happily install it</i> </p><p>Hardness clause: in extraordinary cases, and for 'grandfathered in' packages, we can deviate from this policy, as long as security and maintainability are kept in mind. </p><p><b>apt.wikimedia.org</b> </p><p>We assume that whatever is good for production is also OK for Toolforge. </p><p><b>aptly</b> </p><p>We manage the aptly repository ourselves. </p> <ul><li>Packages in aptly need to be built by Toolforge admins <ul><li>we cannot import .deb files from untrusted 3rd party sources</li></ul></li></ul> <ul><li>Package source files need to come from a trusted source <ul><li>a source file from a trusted source (i.e. backports), or</li> <li>we build the debian source files ourselves</li> <li>we cannot build .dcs files from untrusted 3rd party sources</li></ul></li></ul> <ul><li>Packages need to be easy to update and build <ul><li>cowbuilder/pdebuild OK</li> <li>fpm is OK</li> <li>See <a href="/wiki/Nova_Resource:Tools/Admin/Deploy_new_jobutils_package" class="mw-redirect" title="Nova Resource:Tools/Admin/Deploy new jobutils package">Deploy new jobutils package</a> for an example walk through of building and adding packages to aptly.</li></ul></li></ul> <ul><li>We only package if strictly necessary <ul><li>infrastructure packages</li> <li>packages that should be available for effective development (e.g. composer or sbt)</li> <li>not: python-*, lib*-perl, ..., which should just be installed with the available platform-specific package managers</li></ul></li></ul> <ul><li>For each package, it should be clear who is responsible for keeping it up to date <ul><li>for infrastructure packages, this should be one of the paid staffers</li></ul></li></ul> <p>A list of locally maintained packages can be found under <a href="/wiki/Nova_Resource:Tools/Admin/local_packages" class="mw-redirect" title="Nova Resource:Tools/Admin/local packages">/local packages</a>. </p> <div class="mw-heading mw-heading4"><h4 id="Building_packages" data-mw-thread-id="h-Building_packages-Local_package_management"><span data-mw-comment-start="" id="h-Building_packages-Local_package_management"></span>Building packages<span data-mw-comment-end="h-Building_packages-Local_package_management"></span></h4></div> <style data-mw-deduplicate="TemplateStyles:r2211903">.mw-parser-output .note{background-position:left 7px top 50%;padding:0.5em 0.5em 0.5em 40px;margin:0.5em 0;overflow:hidden;background-color:#f8f9fa;color:#333;background-repeat:no-repeat;border:1px solid #ddd}.mw-parser-output .note-inline{display:inline-block;vertical-align:middle}.mw-parser-output .note-info{background-color:#eaf3ff;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg");background-size:25px;border-color:#a3caff;padding-left:40px;min-height:25px}.mw-parser-output .note-reminder{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/a/a8/OOjs_UI_icon_lightbulb-yellow.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-warn{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/3/3b/OOjs_UI_icon_alert-warning.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-error{background-color:#fee7e6;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/b/bf/OOjs_UI_icon_notice-destructive.svg");background-size:25px;border-color:#c33;min-height:25px}@media screen{html.skin-theme-clientpref-night .mw-parser-output .note{background-color:transparent;color:inherit}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .note{background-color:transparent;color:inherit}}</style><div role="note" class="note note-info"> moved to <a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Portal:Toolforge/Admin/Packaging</a></div> <div class="mw-heading mw-heading4"><h4 id="Deploy_new_misctools_package" data-mw-thread-id="h-Deploy_new_misctools_package-Local_package_management"><span data-mw-comment-start="" id="h-Deploy_new_misctools_package-Local_package_management"></span>Deploy new misctools package<span data-mw-comment-end="h-Deploy_new_misctools_package-Local_package_management"></span></h4></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2211903"><div role="note" class="note note-info"> moved to <a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Portal:Toolforge/Admin/Packaging</a> </div> <div class="mw-heading mw-heading4"><h4 id="Testing/QA_for_a_new_tools-webservice_package" data-mw-thread-id="h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management"><span id="Testing.2FQA_for_a_new_tools-webservice_package"></span><span data-mw-comment-start="" id="h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management"></span>Testing/QA for a new tools-webservice package<span data-mw-comment-end="h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management"></span></h4></div> <p>See also <a class="external text" href="https://gerrit.wikimedia.org/r/plugins/gitiles/operations/software/tools-webservice/+/refs/heads/master/README">tools-webservice source tree README</a>. </p><p>There is a simple flask app in toolsbeta using the tool <code>test</code> that is set up to be deployed via webservice on Kubernetes. </p><p>After running <code>become test</code>, you can go to the <code>qa/tools-webservice</code> directory. This is checked out via anonymous https, and is suitable for checking out a patch you are reviewing. There is an untracked file in there that is useful here, usually. The webservicefile at the route is just a copy of the one in the <code>scripts</code> folder in the repo. The only difference is: </p> <div class="mw-highlight mw-highlight-lang-diff mw-content-ltr" dir="ltr"><pre><span></span><span class="gu">9d8</span> <span class="gd">< sys.path.insert(0, '')</span> </pre></div> <p>That exchanges the distribution installed package in the python path for the local directory, so if you run <code>./webservice $somecommand</code> it will run what is in your local folder rather than what is in <code>/usr/lib/python3/dist-packages/</code>. If you are testing changes made directly to <code>scripts/webservice</code> in the repo, you will likely need to copy that over the file and add <code>sys.path.insert(0, "")</code> after the import sys line. </p><p>If there is no <code>import sys</code> line in this version of the code, add one! This should let you bang on your new version without having to mess with packaging, yet. </p> <div class="mw-heading mw-heading4"><h4 id="Deploy_new_tools-webservice_package" data-mw-thread-id="h-Deploy_new_tools-webservice_package-Local_package_management"><span data-mw-comment-start="" id="h-Deploy_new_tools-webservice_package-Local_package_management"></span>Deploy new tools-webservice package<span data-mw-comment-end="h-Deploy_new_tools-webservice_package-Local_package_management"></span></h4></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2211903"><div role="note" class="note note-info"> moved to <a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Portal:Toolforge/Admin/Packaging</a> </div> <div class="mw-heading mw-heading3"><h3 id="Webserver_statistics" data-mw-thread-id="h-Webserver_statistics-Administrative_tasks"><span data-mw-comment-start="" id="h-Webserver_statistics-Administrative_tasks"></span>Webserver statistics<span data-mw-comment-end="h-Webserver_statistics-Administrative_tasks"></span></h3></div> <p>To get a look at webserver statistics, <a rel="nofollow" class="external text" href="//goaccess.io">goaccess</a> is installed on the webproxies. Usage: </p> <pre>goaccess --date-format="%d/%b/%Y" --log-format='%h - - [%d:%t %^] "%r" %s %b "%R" "%u"' -q -f/var/log/nginx/access.log</pre> <p>Interactive key bindings are documented on <a rel="nofollow" class="external text" href="http://goaccess.io/man#interactive-keys">the man page</a>. HTML output is supported by piping to a file. Note that nginx logs are rotated (twice?) daily, so there is only very recent data available. </p> <div class="mw-heading mw-heading3"><h3 id="Banning_an_IP_from_tool_labs" data-mw-thread-id="h-Banning_an_IP_from_tool_labs-Administrative_tasks"><span data-mw-comment-start="" id="h-Banning_an_IP_from_tool_labs-Administrative_tasks"></span>Banning an IP from tool labs<span data-mw-comment-end="h-Banning_an_IP_from_tool_labs-Administrative_tasks"></span></h3></div> <p>On <a href="/w/index.php?title=Hiera:Tools&action=edit&redlink=1" class="new" title="Hiera:Tools (page does not exist)">Hiera:Tools</a>, add the IP to the list of dynamicproxy::banned_ips, then force a puppet run on the webproxies. Add a note to <a href="/wiki/Help:Toolforge/Banned" title="Help:Toolforge/Banned">Help:Toolforge/Banned</a> explaining why. The user will get a message like <a class="external autonumber" href="https://toolforge.org/.error/banned.html">[1]</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Deploying_the_main_web_page" data-mw-thread-id="h-Deploying_the_main_web_page-Administrative_tasks"><span data-mw-comment-start="" id="h-Deploying_the_main_web_page-Administrative_tasks"></span>Deploying the main web page<span data-mw-comment-end="h-Deploying_the_main_web_page-Administrative_tasks"></span></h3></div><p> This website (plus the 403/500/503 error pages) are hosted under <code>tools.admin</code>. To deploy,</p><div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>become<span class="w"> </span>admin <span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span>tool-admin-web <span class="gp">$ </span>git<span class="w"> </span>pull </pre></div> <div class="mw-heading mw-heading3"><h3 id="Regenerate_replica.my.cnf" data-mw-thread-id="h-Regenerate_replica.my.cnf-Administrative_tasks"><span data-mw-comment-start="" id="h-Regenerate_replica.my.cnf-Administrative_tasks"></span>Regenerate replica.my.cnf<span data-mw-comment-end="h-Regenerate_replica.my.cnf-Administrative_tasks"></span></h3></div> <style data-mw-deduplicate="TemplateStyles:r2199624">.mw-parser-output .mw-tpl-rellink{font-style:italic;padding-bottom:0.4em;padding-left:0;margin-bottom:0.4em;color:#555;border-bottom:1px solid var(--border-color-base,#a2a9b1)}</style> <div class="rellink mw-tpl-rellink">See also: <a href="/wiki/Portal:Data_Services/Admin/Shared_storage#maintain-dbusers" title="Portal:Data Services/Admin/Shared storage">Portal:Data Services/Admin/Shared storage#maintain-dbusers</a></div> <p>This requires access to the cloudcontrol host <a rel="nofollow" class="external text" href="https://codesearch.wmcloud.org/search/?q=maintain_dbusers_primary&files=&excludeFiles=&repos=">which is running maintain-kubeusers</a>, and can be done as follows: </p> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>cloudcontrolXXXX.wikimedia.org <span class="gp">$ </span>sudo<span class="w"> </span>/usr/local/sbin/maintain-dbusers<span class="w"> </span>delete<span class="w"> </span>tools.<span class="si">${</span><span class="nv">NAME</span><span class="si">}</span><span class="w"> </span>--account-type<span class="o">=</span>tool <span class="go">:# or</span> <span class="gp">$ </span>sudo<span class="w"> </span>/usr/local/sbin/maintain-dbusers<span class="w"> </span>delete<span class="w"> </span><span class="si">${</span><span class="nv">USERNAME</span><span class="si">}</span><span class="w"> </span>--account-type<span class="o">=</span>user </pre></div> <p>Once the account has been deleted, the maintain-dbusers service will automatically recreate the user account. </p> <div class="mw-heading mw-heading4"><h4 id="Debugging_bad_MariaDB_credentials" data-mw-thread-id="h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf"><span data-mw-comment-start="" id="h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf"></span>Debugging bad MariaDB credentials<span data-mw-comment-end="h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf"></span></h4></div> <p><span id="Debugging_bad_mysql_credentials"></span> Sometimes things go wrong and a user's <code>replica.my.cnf</code> credentials don't propigate everywhere. You can check the status on various servers to try and narrow down what went wrong. </p><p>The database credentials needed are in <code>/etc/dbusers.yaml</code> on the cloudcontrol host running maintain-dbusrs. </p> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>cloudcontrolXXXX.wikimedia.org <span class="gp">$ </span>sudo<span class="w"> </span>cat<span class="w"> </span>/etc/dbusers.yaml <span class="go">:# look for the accounts-backend['password'] for the m5-master connections (user: labsdbaccounts)</span> <span class="go">:# look for the labsdbs['password'] for the other connections (user: labsdbadmin)</span> <span class="gp">$ </span><span class="nv">CHECK_UID</span><span class="o">=</span>u12345<span class="w"> </span><span class="c1"># User id to check for</span> <span class="go">:# Check if the user is in our meta datastore</span> <span class="gp">$ </span>mariadb<span class="w"> </span>-h<span class="w"> </span>m5-master.eqiad.wmnet<span class="w"> </span>-u<span class="w"> </span>labsdbaccounts<span class="w"> </span>-p<span class="w"> </span>-e<span class="w"> </span><span class="s2">"USE labsdbaccounts; SELECT * FROM account WHERE mysql_username='</span><span class="si">${</span><span class="nv">CHECK_UID</span><span class="si">}</span><span class="s2">'\G"</span> <span class="go">:# Check if all the accounts are created in the labsdb boxes from meta datastore.</span> <span class="gp">$ </span><span class="nv">ACCT_ID</span><span class="o">=</span>....<span class="w"> </span><span class="c1"># Account_id is foreign key (id from account table)</span> <span class="gp">$ </span>mariadb<span class="w"> </span>-h<span class="w"> </span>m5-master.eqiad.wmnet<span class="w"> </span>-u<span class="w"> </span>labsdbaccounts<span class="w"> </span>-p<span class="w"> </span>-e<span class="w"> </span><span class="s2">"USE labsdbaccounts; SELECT * FROM labsdbaccounts.account_host WHERE account_id=</span><span class="si">${</span><span class="nv">ACCT_ID</span><span class="si">}</span><span class="s2">\G"</span> <span class="go">:# Check the actual labsdbs if needed</span> <span class="gp">$ </span>mariadb<span class="w"> </span>-h<span class="w"> </span>clouddbXXXX.eqiad.wmnet<span class="w"> </span>-u<span class="w"> </span>labsdbadmin<span class="w"> </span>-p<span class="w"> </span>-e<span class="w"> </span><span class="s1">'SELECT User, Password from mysql.user where User like "${CHECK_UID}";'</span> <span class="go">:# Resynchronize account state on the replicas by finding missing GRANTS on each db server</span> <span class="gp">$ </span>sudo<span class="w"> </span>maintain-dbusers<span class="w"> </span>harvest-replicas </pre></div> <p>See <a href="https://phabricator.wikimedia.org/T183644" class="extiw" title="phab:T183644">phab:T183644</a> for an example of fixing automatic credential creation caused when a old LDAP user becomes a Toolforge member and has an untracked user account on toolsdb. </p> <div class="mw-heading mw-heading3"><h3 id="Regenerate_kubernetes_credentials_for_tools_(.kube/config)" data-mw-thread-id="h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks"><span id="Regenerate_kubernetes_credentials_for_tools_.28.kube.2Fconfig.29"></span><span data-mw-comment-start="" id="h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks"></span>Regenerate kubernetes credentials for tools (.kube/config)<span data-mw-comment-end="h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks"></span></h3></div> <p>With admin credentials (root on a control plane node will do), run <code>kubectl -n tool-<toolname> delete cm maintain-kubeusers-<toolname></code>; it should get regenerated within minutes. </p> <div class="mw-heading mw-heading3"><h3 id="Adding_K8S_Components" data-mw-thread-id="h-Adding_K8S_Components-Administrative_tasks"><span data-mw-comment-start="" id="h-Adding_K8S_Components-Administrative_tasks"></span>Adding K8S Components<span data-mw-comment-end="h-Adding_K8S_Components-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes#Building_new_nodes" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes#Building_new_nodes</a> </p> <div class="mw-heading mw-heading3"><h3 id="Deleting_a_tool" data-mw-thread-id="h-Deleting_a_tool-Administrative_tasks"><span data-mw-comment-start="" id="h-Deleting_a_tool-Administrative_tasks"></span>Deleting a tool<span data-mw-comment-end="h-Deleting_a_tool-Administrative_tasks"></span></h3></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199615"> <div style="float:right; clear:right; margin-left:1em;" class="plainlinks tracked mw-trackedTemplate">Tracked in <a href="https://phabricator.wikimedia.org/" class="extiw" title="phabricator:">Phabricator</a><br/><b><a href="https://phabricator.wikimedia.org/T170355" class="extiw" title="phabricator:T170355"><span class="trakfab-T170355">Task T170355</span></a></b> <span class="tracked-status tracked-status--resolved">Resolved</span></div> <p>For batch or CLI deletion of tools, use the 'mark_tool' command on a cloudcontrol node: </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Tool_disable_process.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/labs/thumb/a/a5/Tool_disable_process.png/220px-Tool_disable_process.png" decoding="async" width="220" height="186" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/labs/thumb/a/a5/Tool_disable_process.png/330px-Tool_disable_process.png 1.5x, //upload.wikimedia.org/wikipedia/labs/thumb/a/a5/Tool_disable_process.png/440px-Tool_disable_process.png 2x" data-file-width="1671" data-file-height="1411"/></a><figcaption>The awful truth about tool deletion</figcaption></figure> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">andrew@cloudcontrol1003:~$ </span>sudo<span class="w"> </span>mark_tool <span class="go">usage: mark_tool [-h] [--ldap-user LDAP_USER] [--ldap-password LDAP_PASSWORD]</span> <span class="go"> [--ldap-base-dn LDAP_BASE_DN] [--project PROJECT] [--disable]</span> <span class="go"> [--delete] [--enable]</span> <span class="go"> tool</span> <span class="go">mark_tool: error: the following arguments are required: tool</span> </pre></div> <p>Maintainers can mark their tools for deletion using the "Disable tool" button on the tool's detail page on <a class="external free" href="https://toolsadmin.wikimedia.org/">https://toolsadmin.wikimedia.org/</a>. In either case, the immediate effect of disabling a tool is to stop any running jobs, prevent users from logging in as that tool, and schedule archiving and deletion for 40 days in the future. </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Tool_restore_process.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/labs/thumb/7/75/Tool_restore_process.png/220px-Tool_restore_process.png" decoding="async" width="220" height="122" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/labs/thumb/7/75/Tool_restore_process.png/330px-Tool_restore_process.png 1.5x, //upload.wikimedia.org/wikipedia/labs/thumb/7/75/Tool_restore_process.png/440px-Tool_restore_process.png 2x" data-file-width="1671" data-file-height="930"/></a><figcaption>A tool can be restored within 40 days of being disabled</figcaption></figure> <p>Tool archives are stored on the tools NFS server, currently <code>tools-nfs-2.tools.eqiad1.wikimedia.cloud</code>: </p> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">root@labstore1004:/srv/disable-tool# </span>ls<span class="w"> </span>-ltrah<span class="w"> </span>/srv/tools/archivedtools/ <span class="go">total 1.8G</span> <span class="go">drwxr-xr-x 5 root root 4.0K Jun 21 19:37 ..</span> <span class="go">-rw-r--r-- 1 root root 102K Jul 22 22:15 andrewtesttooltwo</span> <span class="go">-rw-r--r-- 1 root root 45 Oct 13 00:47 andrewtesttooltwo.tgz</span> <span class="go">-rw-r--r-- 1 root root 8.3M Oct 13 03:20 mediaplaycounts.tgz</span> <span class="go">-rw-r--r-- 1 root root 1.8G Oct 13 04:01 projanalysis.tgz</span> <span class="go">-rw-r--r-- 1 root root 1.3M Oct 13 21:05 reportsbot.tgz</span> <span class="go">drwxr-xr-x 2 root root 4.0K Oct 13 21:10 .</span> <span class="go">-rw-r--r-- 1 root root 719K Oct 13 21:10 wsm.tgz</span> <span class="go">-rw-r--r-- 1 root root 4.8K Oct 13 21:20 andrewtesttoolfour.tgz</span> </pre></div> <p>The actual deletion process is shockingly complicated. A tool will only be archived and deleted if all of the prior steps succeed, but disabling of a tool should be a sure thing. </p> <div class="mw-heading mw-heading3"><h3 id="SSL_certificates" data-mw-thread-id="h-SSL_certificates-Administrative_tasks"><span data-mw-comment-start="" id="h-SSL_certificates-Administrative_tasks"></span>SSL certificates<span data-mw-comment-end="h-SSL_certificates-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/SSL_certificates" title="Portal:Toolforge/Admin/SSL certificates">Portal:Toolforge/Admin/SSL_certificates</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Granting_a_tool_write_access_to_Elasticsearch" data-mw-thread-id="h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks"><span data-mw-comment-start="" id="h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks"></span>Granting a tool write access to Elasticsearch<span data-mw-comment-end="h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks"></span></h3></div> <ul><li>Generate a random password and the mkpassword crypt entry for it using the script <a href="https://phabricator.wikimedia.org/P4372" class="extiw" title="phab:P4372">new-es-password.sh</a>. (Must be run a host with the `mkpasswd` command installed. (The mkpasswd is part of the whois Debian package.)</li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>./new-es-password.sh<span class="w"> </span>tools.example <span class="go">tools.example elasticsearch.ini</span> <span class="go">----</span> <span class="go">[elasticsearch]</span> <span class="go">user=tools.example </span> <span class="go">password=A3rJqgFKxa/x4NlnIhmw2cXcV92it/Zv0Yt+a7yhxCw=</span> <span class="go">----</span> <span class="go">tools.example puppet master private (hieradata/labs/tools/common.yaml)</span> <span class="go">----</span> <span class="go">profile::toolforge::elasticsearch::haproxy::elastic_users:</span> <span class="go"> - name: 'tools.example'</span> <span class="go"> password: '$6$FYwP3wxT4K7O9EE$OA3P5972NWJVG/WUnD240sal34/dsNabbcawItevMYO9uoR.fJBrjSABex0EDW0wlkWHID1Tf4oJoiNvYFGmy/'</span> </pre></div> <ul><li>Add the private SHA512 hash to the <a href="/wiki/Portal:Toolforge/Nodes#misc_nodes" title="Portal:Toolforge/Nodes">tools puppetmaster</a>:</li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>tools-puppetserver-01.tools.eqiad1.wikimedia.cloud <span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span>/srv/git/labs/private <span class="gp">$ </span>sudo<span class="w"> </span>-i<span class="w"> </span>vim<span class="w"> </span>/var/lib/git/labs/private/hieradata/labs/tools/common.yaml <span class="go">... merge the hiera data with the existing key...</span> <span class="go">:wq</span> <span class="gp">$ </span>sudo<span class="w"> </span>git<span class="w"> </span>add<span class="w"> </span>hieradata/labs/tools/common.yaml <span class="gp">$ </span>sudo<span class="w"> </span>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">"[local] Elasticsearch credentials for </span><span class="nv">$TOOL</span><span class="s2">"</span> </pre></div> <ul><li>Force a puppet run on tools-elastic nodes using <a href="/wiki/Cumin" title="Cumin">Cumin</a></li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">cloudcumin1001.eqiad.wmnet:~$ </span>sudo<span class="w"> </span>cumin<span class="w"> </span><span class="s2">"O{project:tools name:.*elastic.*}"</span><span class="w"> </span><span class="s2">"run-puppet-agent"</span> </pre></div> <ul><li>Make the credentials available to the tool as <a href="/wiki/Help:Toolforge/Envvars_Service" class="mw-redirect" title="Help:Toolforge/Envvars Service">envvars</a>:</li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>dev.toolforge.org <span class="gp">$ </span>sudo<span class="w"> </span>-i<span class="w"> </span>become<span class="w"> </span>example-tool <span class="gp">$ </span>toolforge<span class="w"> </span>envvars<span class="w"> </span>create<span class="w"> </span>TOOL_ELASTICSEARCH_USER <span class="go">Enter the value of your envvar (prompt is hidden, hit Ctrl+C to abort): <insert user></span> <span class="gp">$ </span>toolforge<span class="w"> </span>envvars<span class="w"> </span>create<span class="w"> </span>TOOL_ELASTICSEARCH_PASSWORD <span class="go">Enter the value of your envvar (prompt is hidden, hit Ctrl+C to abort): <insert password></span> </pre></div> <p><b>Note:</b> An older procedure placed the credentials in <code>/data/project/$TOOL/.elasticsearch.ini</code> instead. </p> <ul><li>Resolve the ticket!</li></ul> <div class="mw-heading mw-heading3"><h3 id="Package_upgrades" data-mw-thread-id="h-Package_upgrades-Administrative_tasks"><span data-mw-comment-start="" id="h-Package_upgrades-Administrative_tasks"></span>Package upgrades<span data-mw-comment-end="h-Package_upgrades-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Cloud_VPS/Admin/Managing_package_upgrades" class="mw-redirect" title="Portal:Cloud VPS/Admin/Managing package upgrades">Managing package upgrades</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)" data-mw-thread-id="h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks"><span id="Creating_a_new_Docker_image_.28e.g._for_new_versions_of_Node.js.29"></span><span data-mw-comment-start="" id="h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks"></span>Creating a new Docker image (e.g. for new versions of Node.js)<span data-mw-comment-end="h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes#Docker_Images" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes#Docker_Images</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Kubernetes" data-mw-thread-id="h-Kubernetes"><span data-mw-comment-start="" id="h-Kubernetes"></span>Kubernetes<span data-mw-comment-end="h-Kubernetes"></span></h2></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Build_service" data-mw-thread-id="h-Build_service"><span data-mw-comment-start="" id="h-Build_service"></span>Build service<span data-mw-comment-end="h-Build_service"></span></h2></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Build_Service" title="Portal:Toolforge/Admin/Build Service">Portal:Toolforge/Admin/Build_Service</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Tools-mail_/_Exim" data-mw-thread-id="h-Tools-mail_/_Exim"><span id="Tools-mail_.2F_Exim"></span><span data-mw-comment-start="" id="h-Tools-mail_/_Exim"></span>Tools-mail / Exim<span data-mw-comment-end="h-Tools-mail_/_Exim"></span></h2></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Exim" title="Portal:Toolforge/Admin/Exim">Portal:Toolforge/Admin/Exim</a> and <a href="/wiki/Portal:Cloud_VPS/Admin/Email#Operations" title="Portal:Cloud VPS/Admin/Email">Portal:Cloud_VPS/Admin/Email#Operations</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Users_and_community" data-mw-thread-id="h-Users_and_community"><span data-mw-comment-start="" id="h-Users_and_community"></span>Users and community<span data-mw-comment-end="h-Users_and_community"></span></h2></div> <p>Some information about how to manage users and general community and their relationship with Toolforge. </p> <div class="mw-heading mw-heading3"><h3 id="Project_membership_request_approval" data-mw-thread-id="h-Project_membership_request_approval-Users_and_community"><span data-mw-comment-start="" id="h-Project_membership_request_approval-Users_and_community"></span>Project membership request approval<span data-mw-comment-end="h-Project_membership_request_approval-Users_and_community"></span></h3></div> <p>User access requests show up in <a class="external free" href="https://toolsadmin.wikimedia.org/tools/membership/">https://toolsadmin.wikimedia.org/tools/membership/</a> </p><p>Some guidelines for account approvals, based on <a href="https://phabricator.wikimedia.org/T128158#2132893" class="extiw" title="phab:T128158">advice from scfc</a>: </p> <ol><li>If the request contains any defamatory or abusive information as part of the username(s), reason, or comments → mark as <b>Declined</b> and check the "Suppress this request (hide from non-admin users)" checkbox. <ul><li>You should also block the user on Wikitech and consider contacting a <a href="https://meta.wikimedia.org/wiki/Stewards" class="extiw" title="meta:Stewards">Steward</a> for wider review of the SUL account.</li></ul></li> <li>If the user name "looks" like a bot or someone else who could not consent to the <a href="/wiki/Wikitech:Cloud_Services_Terms_of_use" title="Wikitech:Cloud Services Terms of use">Terms of use</a> and <a href="/wiki/Help:Toolforge/Rules" title="Help:Toolforge/Rules">Rules</a> → mark as <b>Declined</b>.</li> <li>Check the status of the associated <a href="https://meta.wikimedia.org/wiki/Help:Unified_login" class="extiw" title="meta:Help:Unified login">SUL account</a>. If the user is banned on one or more wikis → mark as <b>Declined</b>.</li> <li>If the stated purpose is "tangible" ("I want to move my bot x to Labs", "I want to build a web app that does y", etc.) → mark as <b>Approved</b>. <ul><li>If you know that someone else has been working on the same problem, add a message explaining who the user should contact or where they might find more information.</li></ul></li> <li>If the stated purpose is "abstract" ("research", "experimentation", etc.) and there is a hackathon ongoing or planned, the user has a non-throw-away mail address, the user has created a user page with coherent information about theirself or linked a SUL account of good standing, etc. → mark as <b>Approved</b>.</li> <li>Otherwise add a comment asking for clarification of their reason for use and mark as <b>Feedback needed</b>. The request is not really "denied", but more (indefinitely) "delayed".</li></ol> <p>Requests left in <b>Feedback needed</b> for more information for more than 30 days should usually be declined with a message like "Feel free to apply again later with more complete information." </p> <div class="mw-heading mw-heading3"><h3 id="Quota_management" data-mw-thread-id="h-Quota_management-Users_and_community"><span data-mw-comment-start="" id="h-Quota_management-Users_and_community"></span>Quota management<span data-mw-comment-end="h-Quota_management-Users_and_community"></span></h3></div> <p>Toolforge quotas are managed via maintain-kubeusers. </p> <ul><li>Have the user open a phabricator ticket, for the papertrail. See also <a href="/wiki/Help:Toolforge/Kubernetes#Quotas_and_Resources" title="Help:Toolforge/Kubernetes">Help:Toolforge/Kubernetes#Quotas_and_Resources</a></li></ul> <ul><li>Send a patch for maintain-kubeusers, have it reviewed and merged:</li></ul> <p><a class="external free" href="https://gitlab.wikimedia.org/repos/cloud/toolforge/toolforge-deploy/-/blob/main/components/maintain-kubeusers/values/tools.yaml">https://gitlab.wikimedia.org/repos/cloud/toolforge/toolforge-deploy/-/blob/main/components/maintain-kubeusers/values/tools.yaml</a> </p> <ul><li>Deploy in the cluster, using the deploy <a href="/wiki/Portal:Cloud_VPS/Admin/Cookbooks" title="Portal:Cloud VPS/Admin/Cookbooks">Portal:Cloud_VPS/Admin/Cookbooks</a></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Other" data-mw-thread-id="h-Other"><span data-mw-comment-start="" id="h-Other"></span>Other<span data-mw-comment-end="h-Other"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="How_do_Toolforge_web_services_actually_work?" data-mw-thread-id="h-How_do_Toolforge_web_services_actually_work?-Other"><span id="How_do_Toolforge_web_services_actually_work.3F"></span><span data-mw-comment-start="" id="h-How_do_Toolforge_web_services_actually_work?-Other"></span>How do Toolforge web services actually work?<span data-mw-comment-end="h-How_do_Toolforge_web_services_actually_work?-Other"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes#Ingress" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes#Ingress</a> </p> <div class="mw-heading mw-heading3"><h3 id="What_makes_a_root/Giving_root_access" data-mw-thread-id="h-What_makes_a_root/Giving_root_access-Other"><span id="What_makes_a_root.2FGiving_root_access"></span><span data-mw-comment-start="" id="h-What_makes_a_root/Giving_root_access-Other"></span>What makes a root/Giving root access<span data-mw-comment-end="h-What_makes_a_root/Giving_root_access-Other"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a> </p> <div class="mw-heading mw-heading3"><h3 id="Servicegroup_log" data-mw-thread-id="h-Servicegroup_log-Other"><span data-mw-comment-start="" id="h-Servicegroup_log-Other"></span>Servicegroup log<span data-mw-comment-end="h-Servicegroup_log-Other"></span></h3></div> <p>tools.admin runs <code>/data/project/admin/bin/toolhistory</code>, which provides an hourly snapshot of <code>ldaplist -l servicegroup</code> as git repository in <code>/data/project/admin/var/lib/git/servicegroups</code> </p> <div class="mw-heading mw-heading3"><h3 id="Useful_administrative_tools" data-mw-thread-id="h-Useful_administrative_tools-Other"><span data-mw-comment-start="" id="h-Useful_administrative_tools-Other"></span>Useful administrative tools<span data-mw-comment-end="h-Useful_administrative_tools-Other"></span></h3></div> <p>These tools offer useful information about Toolforge itself: </p> <ul><li><a href="https://iw.toolforge.org/tool-db-usage/" class="extiw" title="toolforge:tool-db-usage/">ToolsDB</a> - Statistics about tables owned by tools</li> <li><a href="https://iw.toolforge.org/k8s-stats" class="extiw" title="toolforge:k8s-stats">k8s-stats</a> - examine what our tools are doing</li> <li><a href="https://iw.toolforge.org/openstack-browser/project/tools" class="extiw" title="toolforge:openstack-browser/project/tools">OpenStack Browser</a> - examine projects, instances, web proxies, and Puppet config</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Brainstorming" data-mw-thread-id="h-Brainstorming"><span data-mw-comment-start="" id="h-Brainstorming"></span>Brainstorming<span data-mw-comment-end="h-Brainstorming"></span></h2></div> <ul><li><a href="/wiki/Portal:Toolforge/Admin/BotLicensing" class="mw-redirect" title="Portal:Toolforge/Admin/BotLicensing">/BotLicensing</a></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Sub_pages" data-mw-thread-id="h-Sub_pages"><span data-mw-comment-start="" id="h-Sub_pages"></span>Sub pages<span data-mw-comment-end="h-Sub_pages"></span></h2></div> <div class="mw-prefixindex-body"><ul class="mw-prefixindex-list"><li><a href="/wiki/Portal:Toolforge/Admin/APIs" title="Portal:Toolforge/Admin/APIs">APIs</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Apt_repository" title="Portal:Toolforge/Admin/Apt repository">Apt repository</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Archive" title="Portal:Toolforge/Admin/Archive">Archive</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Build_Service" title="Portal:Toolforge/Admin/Build Service">Build Service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Buildpacks" title="Portal:Toolforge/Admin/Buildpacks">Buildpacks</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Envvars_Service" title="Portal:Toolforge/Admin/Envvars Service">Envvars Service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Exim" title="Portal:Toolforge/Admin/Exim">Exim</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Harbor" title="Portal:Toolforge/Admin/Harbor">Harbor</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Harbor/maintain-harbor" title="Portal:Toolforge/Admin/Harbor/maintain-harbor">Harbor/maintain-harbor</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Infrastructure_tools" title="Portal:Toolforge/Admin/Infrastructure tools">Infrastructure tools</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Jobs_framework" title="Portal:Toolforge/Admin/Jobs framework">Jobs framework</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes" title="Portal:Toolforge/Admin/Kubernetes">Kubernetes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/2020_Kubernetes_cluster_rebuild_plan_notes" title="Portal:Toolforge/Admin/Kubernetes/2020 Kubernetes cluster rebuild plan notes">Kubernetes/2020 Kubernetes cluster rebuild plan notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Certificates" title="Portal:Toolforge/Admin/Kubernetes/Certificates">Kubernetes/Certificates</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Docker-registry" title="Portal:Toolforge/Admin/Kubernetes/Docker-registry">Kubernetes/Docker-registry</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Etcd_(deprecated)" title="Portal:Toolforge/Admin/Kubernetes/Etcd (deprecated)">Kubernetes/Etcd (deprecated)</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Labels" title="Portal:Toolforge/Admin/Kubernetes/Labels">Kubernetes/Labels</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Networking_and_ingress" title="Portal:Toolforge/Admin/Kubernetes/Networking and ingress">Kubernetes/Networking and ingress</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/New_cluster" title="Portal:Toolforge/Admin/Kubernetes/New cluster">Kubernetes/New cluster</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Pod_tracing" title="Portal:Toolforge/Admin/Kubernetes/Pod tracing">Kubernetes/Pod tracing</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/RBAC_and_Pod_security" title="Portal:Toolforge/Admin/Kubernetes/RBAC and Pod security">Kubernetes/RBAC and Pod security</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/RBAC_and_Pod_security/PSP_migration" title="Portal:Toolforge/Admin/Kubernetes/RBAC and Pod security/PSP migration">Kubernetes/RBAC and Pod security/PSP migration</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes">Kubernetes/Upgrading Kubernetes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.21_to_1.22_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.21 to 1.22 notes">Kubernetes/Upgrading Kubernetes/1.21 to 1.22 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.22_to_1.23_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.22 to 1.23 notes">Kubernetes/Upgrading Kubernetes/1.22 to 1.23 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.24_to_1.25_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.24 to 1.25 notes">Kubernetes/Upgrading Kubernetes/1.24 to 1.25 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.25_to_1.26_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.25 to 1.26 notes">Kubernetes/Upgrading Kubernetes/1.25 to 1.26 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.26_to_1.27_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.26 to 1.27 notes">Kubernetes/Upgrading Kubernetes/1.26 to 1.27 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.27_to_1.28_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.27 to 1.28 notes">Kubernetes/Upgrading Kubernetes/1.27 to 1.28 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/foxtrot-ldap" title="Portal:Toolforge/Admin/Kubernetes/foxtrot-ldap">Kubernetes/foxtrot-ldap</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/lima-kilo" title="Portal:Toolforge/Admin/Kubernetes/lima-kilo">Kubernetes/lima-kilo</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Legacy_redirector_for_webservices" title="Portal:Toolforge/Admin/Legacy redirector for webservices">Legacy redirector for webservices</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Maintenance" title="Portal:Toolforge/Admin/Maintenance">Maintenance</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting" title="Portal:Toolforge/Admin/Monthly meeting">Monthly meeting</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2022-11-15" title="Portal:Toolforge/Admin/Monthly meeting/2022-11-15">Monthly meeting/2022-11-15</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2022-12-13" title="Portal:Toolforge/Admin/Monthly meeting/2022-12-13">Monthly meeting/2022-12-13</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-01-31" title="Portal:Toolforge/Admin/Monthly meeting/2023-01-31">Monthly meeting/2023-01-31</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-02-21" title="Portal:Toolforge/Admin/Monthly meeting/2023-02-21">Monthly meeting/2023-02-21</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-04-04" title="Portal:Toolforge/Admin/Monthly meeting/2023-04-04">Monthly meeting/2023-04-04</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-05-02" title="Portal:Toolforge/Admin/Monthly meeting/2023-05-02">Monthly meeting/2023-05-02</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-06-06" title="Portal:Toolforge/Admin/Monthly meeting/2023-06-06">Monthly meeting/2023-06-06</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-07-11" title="Portal:Toolforge/Admin/Monthly meeting/2023-07-11">Monthly meeting/2023-07-11</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-09-05" title="Portal:Toolforge/Admin/Monthly meeting/2023-09-05">Monthly meeting/2023-09-05</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-10-03" title="Portal:Toolforge/Admin/Monthly meeting/2023-10-03">Monthly meeting/2023-10-03</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-11-07" title="Portal:Toolforge/Admin/Monthly meeting/2023-11-07">Monthly meeting/2023-11-07</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-12-19" title="Portal:Toolforge/Admin/Monthly meeting/2023-12-19">Monthly meeting/2023-12-19</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-01-16" title="Portal:Toolforge/Admin/Monthly meeting/2024-01-16">Monthly meeting/2024-01-16</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-02-13" title="Portal:Toolforge/Admin/Monthly meeting/2024-02-13">Monthly meeting/2024-02-13</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-03-12" title="Portal:Toolforge/Admin/Monthly meeting/2024-03-12">Monthly meeting/2024-03-12</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-04-09" title="Portal:Toolforge/Admin/Monthly meeting/2024-04-09">Monthly meeting/2024-04-09</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-05-14" title="Portal:Toolforge/Admin/Monthly meeting/2024-05-14">Monthly meeting/2024-05-14</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-06-25" title="Portal:Toolforge/Admin/Monthly meeting/2024-06-25">Monthly meeting/2024-06-25</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-07-09" title="Portal:Toolforge/Admin/Monthly meeting/2024-07-09">Monthly meeting/2024-07-09</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-09-03" title="Portal:Toolforge/Admin/Monthly meeting/2024-09-03">Monthly meeting/2024-09-03</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-10-01" title="Portal:Toolforge/Admin/Monthly meeting/2024-10-01">Monthly meeting/2024-10-01</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-11-05" title="Portal:Toolforge/Admin/Monthly meeting/2024-11-05">Monthly meeting/2024-11-05</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Packaging</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Prometheus" title="Portal:Toolforge/Admin/Prometheus">Prometheus</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Pywikibot_image" title="Portal:Toolforge/Admin/Pywikibot image">Pywikibot image</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Redis" title="Portal:Toolforge/Admin/Redis">Redis</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks" title="Portal:Toolforge/Admin/Runbooks">Runbooks</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/HarborComponentDown" title="Portal:Toolforge/Admin/Runbooks/HarborComponentDown">Runbooks/HarborComponentDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/HarborDown" title="Portal:Toolforge/Admin/Runbooks/HarborDown">Runbooks/HarborDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/HarborProbeUnknown" title="Portal:Toolforge/Admin/Runbooks/HarborProbeUnknown">Runbooks/HarborProbeUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Kyverno" title="Portal:Toolforge/Admin/Runbooks/Kyverno">Runbooks/Kyverno</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/MaintainKubeusersDown" title="Portal:Toolforge/Admin/Runbooks/MaintainKubeusersDown">Runbooks/MaintainKubeusersDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Redis" title="Portal:Toolforge/Admin/Runbooks/Redis">Runbooks/Redis</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/TektonDown" title="Portal:Toolforge/Admin/Runbooks/TektonDown">Runbooks/TektonDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/TektonUpMetricUnknown" title="Portal:Toolforge/Admin/Runbooks/TektonUpMetricUnknown">Runbooks/TektonUpMetricUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesCapacity" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesCapacity">Runbooks/ToolforgeKubernetesCapacity</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyServerDown" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyServerDown">Runbooks/ToolforgeKubernetesHAproxyServerDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyUnknown" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyUnknown">Runbooks/ToolforgeKubernetesHAproxyUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesNodeNotReady" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesNodeNotReady">Runbooks/ToolforgeKubernetesNodeNotReady</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses">Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Toolforge_Kyverno_low_policy_resources" title="Portal:Toolforge/Admin/Runbooks/Toolforge Kyverno low policy resources">Runbooks/Toolforge Kyverno low policy resources</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Toolforge_Kyverno_no_policy_resources" title="Portal:Toolforge/Admin/Runbooks/Toolforge Kyverno no policy resources">Runbooks/Toolforge Kyverno no policy resources</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Toolforge_Kyverno_unknown_state" title="Portal:Toolforge/Admin/Runbooks/Toolforge Kyverno unknown state">Runbooks/Toolforge Kyverno unknown state</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsDBReplication" title="Portal:Toolforge/Admin/Runbooks/ToolsDBReplication">Runbooks/ToolsDBReplication</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsNFSDown" title="Portal:Toolforge/Admin/Runbooks/ToolsNFSDown">Runbooks/ToolsNFSDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsNfsAlmostFull" title="Portal:Toolforge/Admin/Runbooks/ToolsNfsAlmostFull">Runbooks/ToolsNfsAlmostFull</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsToolsDBWritableState" title="Portal:Toolforge/Admin/Runbooks/ToolsToolsDBWritableState">Runbooks/ToolsToolsDBWritableState</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/k8s-haproxy" title="Portal:Toolforge/Admin/Runbooks/k8s-haproxy">Runbooks/k8s-haproxy</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/SSL_certificates" title="Portal:Toolforge/Admin/SSL certificates">SSL certificates</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker" title="Portal:Toolforge/Admin/Striker">Striker</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker/Build" title="Portal:Toolforge/Admin/Striker/Build">Striker/Build</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker/Deployments" title="Portal:Toolforge/Admin/Striker/Deployments">Striker/Deployments</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker/FAQ" title="Portal:Toolforge/Admin/Striker/FAQ">Striker/FAQ</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/System_Overview" title="Portal:Toolforge/Admin/System Overview">System Overview</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolforge-sync-meeting" title="Portal:Toolforge/Admin/Toolforge-sync-meeting">Toolforge-sync-meeting</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/ToolsDB" title="Portal:Toolforge/Admin/ToolsDB">ToolsDB</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolsbeta" title="Portal:Toolforge/Admin/Toolsbeta">Toolsbeta</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolschecker" title="Portal:Toolforge/Admin/Toolschecker">Toolschecker</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides" title="Portal:Toolforge/Admin/emergency guides">emergency guides</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides/irc_bot_deployment" title="Portal:Toolforge/Admin/emergency guides/irc bot deployment">emergency guides/irc bot deployment</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides/single_tool_webservice" title="Portal:Toolforge/Admin/emergency guides/single tool webservice">emergency guides/single tool webservice</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides/toolforge_down_notification" title="Portal:Toolforge/Admin/emergency guides/toolforge down notification">emergency guides/toolforge down notification</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/local_packages" title="Portal:Toolforge/Admin/local packages">local packages</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/puppet_refactor" title="Portal:Toolforge/Admin/puppet refactor">puppet refactor</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/replagstats" title="Portal:Toolforge/Admin/replagstats">replagstats</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/toolhistory" title="Portal:Toolforge/Admin/toolhistory">toolhistory</a></li> </ul></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://wikitech.wikimedia.org/w/index.php?title=Portal:Toolforge/Admin&oldid=2241510">https://wikitech.wikimedia.org/w/index.php?title=Portal:Toolforge/Admin&oldid=2241510</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Special:Categories" title="Special:Categories">Categories</a>: <ul><li><a href="/wiki/Category:Toolforge_admin" title="Category:Toolforge admin">Toolforge admin</a></li><li><a href="/wiki/Category:Toolforge" title="Category:Toolforge">Toolforge</a></li><li><a href="/wiki/Category:Cloud_Services_admin" title="Category:Cloud Services admin">Cloud Services admin</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 4 November 2024, at 23:27.</li> <li id="footer-info-copyright">Text is available under the <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-ShareAlike License</a>; additional terms may apply. See <a class="external text" href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use">Terms of Use</a> for details.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Main_Page">About Wikitech</a></li> <li id="footer-places-disclaimers"><a href="https://foundation.wikimedia.org/wiki/General_disclaimer">Disclaimers</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/wikitech.wikimedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//wikitech.wikimedia.org/w/index.php?title=Portal:Toolforge/Admin&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-xl5gk","wgBackendResponseTime":353,"wgDiscussionToolsPageThreads":[{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Admin_permissions","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Failover","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-WebProxy-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Static_webserver-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Checker_service-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Redis-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Prometheus-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Services-Failover","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Command_orchestration","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Administrative_tasks","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Logging_in_as_root-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Disabling_all_ssh_logins_except_root-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Complaints_of_bastion_being_slow-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Local_package_management-Administrative_tasks","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Local_package_policy-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Building_packages-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Deploy_new_misctools_package-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Deploy_new_tools-webservice_package-Local_package_management","replies":[]}]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Webserver_statistics-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Banning_an_IP_from_tool_labs-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Deploying_the_main_web_page-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Regenerate_replica.my.cnf-Administrative_tasks","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf","replies":[]}]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Adding_K8S_Components-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Deleting_a_tool-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-SSL_certificates-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Package_upgrades-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Kubernetes","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Build_service","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Tools-mail_/_Exim","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Users_and_community","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Project_membership_request_approval-Users_and_community","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Quota_management-Users_and_community","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Other","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-How_do_Toolforge_web_services_actually_work?-Other","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-What_makes_a_root/Giving_root_access-Other","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Servicegroup_log-Other","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Useful_administrative_tools-Other","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Brainstorming","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Sub_pages","replies":[]}],"wgPageParseReport":{"discussiontools":{"limitreport-timeusage":"0.031"},"limitreport":{"cputime":"0.172","walltime":"0.230","ppvisitednodes":{"value":2685,"limit":1000000},"postexpandincludesize":{"value":13912,"limit":2097152},"templateargumentsize":{"value":5105,"limit":2097152},"expansiondepth":{"value":13,"limit":100},"expensivefunctioncount":{"value":13,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":35855,"limit":5000000},"timingprofile":["100.00% 145.684 1 -total"," 35.56% 51.811 3 Template:Tracked"," 34.14% 49.742 1 Template:Toolforge_nav_admin"," 26.48% 38.578 9 Template:Dir"," 22.89% 33.350 1 Template:Navigation_sidebar"," 14.57% 21.230 9 Template:Pagelang"," 11.15% 16.244 1 Template:Prefix_search"," 10.01% 14.590 3 Template:Note"," 7.71% 11.231 1 Special:Prefixindex/Portal:Toolforge/Admin/"," 6.40% 9.317 1 Template:See_also"]},"scribunto":{"limitreport-timeusage":{"value":"0.004","limit":"10.000"},"limitreport-memusage":{"value":622797,"limit":52428800}},"cachereport":{"origin":"mw-web.codfw.main-f69cdc8f6-xl5gk","timestamp":"20241124032857","ttl":570,"transientcontent":true}}});});</script> </body> </html>