CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Portal:Toolforge/Admin - Wikitech</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-sticky-header-enabled vector-toc-available";var cookie=document.cookie.match(/(?:^|; )labswikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":true,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy", "wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"8b952bd7-f8de-48fb-9cc1-4b273ef828cb","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Portal:Toolforge/Admin","wgTitle":"Portal:Toolforge/Admin","wgCurRevisionId":2256172,"wgRevisionId":2256172,"wgArticleId":21478,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Toolforge","Toolforge admin","Cloud Services admin"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Portal:Toolforge/Admin","wgRelevantArticleId":21478,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikitech","wgCiteReferencePreviewsActive":true,"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgVisualEditor": {"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":20000,"wgEditSubmitButtonLabelPublish":true,"wgDiscussionToolsFeaturesEnabled":{"replytool":true,"newtopictool":true,"sourcemodetoolbar":true,"topicsubscription":false,"autotopicsub":false,"visualenhancements":false,"visualenhancements_reply":false,"visualenhancements_pageframe":false},"wgDiscussionToolsFallbackEditMode":"visual","wgULSPosition":"personal","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"wgSiteNoticeId":"2.0"};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user": "ready","user.options":"loading","ext.inputBox.styles":"ready","ext.pygments":"ready","mediawiki.special":"ready","oojs-ui-core.styles":"ready","oojs-ui.styles.indicators":"ready","mediawiki.widgets.styles":"ready","oojs-ui-core.icons":"ready","mediawiki.htmlform.ooui.styles":"ready","ext.discussionTools.init.styles":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.pt":"ready","ext.dismissableSiteNotice.styles":"ready"};RLPAGEMODULES=["ext.pygments.view","mediawiki.page.media","site","mediawiki.page.ready","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.site","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.discussionTools.init", "ext.eventLogging","ext.wikimediaEvents","ext.uls.interface","ext.checkUser.clientHints","ext.dismissableSiteNotice"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.discussionTools.init.styles%7Cext.dismissableSiteNotice.styles%7Cext.inputBox.styles%7Cext.pygments%7Cext.uls.pt%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cmediawiki.htmlform.ooui.styles%7Cmediawiki.special%7Cmediawiki.widgets.styles%7Coojs-ui-core.icons%2Cstyles%7Coojs-ui.styles.indicators%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.15"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Portal:Toolforge/Admin - Wikitech"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="icon" href="/static/favicon/wikitech.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikitech (en)"> <link rel="EditURI" type="application/rsd+xml" href="//wikitech.wikimedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://wikitech.wikimedia.org/wiki/Portal:Toolforge/Admin"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/"> <link rel="alternate" type="application/atom+xml" title="Wikitech Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="login.wikimedia.org"> </head> <body class="ext-discussiontools-replytool-enabled ext-discussiontools-newtopictool-enabled ext-discussiontools-sourcemodetoolbar-enabled skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Portal_Toolforge_Admin rootpage-Portal_Toolforge skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" title="Main menu" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-Server-admin-log:-Prod" class="mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Server admin log: Prod</span></a></li><li id="n-Admin-log:-RelEng" class="mw-list-item"><a href="/wiki/Release_Engineering/SAL"><span>Admin log: RelEng</span></a></li><li id="n-Incident-status" class="mw-list-item"><a href="/wiki/Incident_status"><span>Incident status</span></a></li><li id="n-Deployments" class="mw-list-item"><a href="/wiki/Deployments"><span>Deployments</span></a></li><li id="n-SRE-Team-Help" class="mw-list-item"><a href="/wiki/SRE/SRE_Team_requests"><span>SRE Team Help</span></a></li> </ul> </div> </div> <div id="p-Cloud_VPS_&_Toolforge" class="vector-menu mw-portlet mw-portlet-Cloud_VPS_Toolforge" > <div class="vector-menu-heading"> Cloud VPS & Toolforge </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-Cloud-VPS-portal" class="mw-list-item"><a href="/wiki/Portal:Cloud_VPS"><span>Cloud VPS portal</span></a></li><li id="n-Toolforge-portal" class="mw-list-item"><a href="/wiki/Portal:Toolforge"><span>Toolforge portal</span></a></li><li id="n-Request-VPS-project" class="mw-list-item"><a href="https://phabricator.wikimedia.org/project/view/2875/"><span>Request VPS project</span></a></li><li id="n-Admin-log:-Cloud-VPS" class="mw-list-item"><a href="/wiki/Cloud_VPS_Server_Admin_Log"><span>Admin log: Cloud VPS</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikitech.svg" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikitech" src="/static/images/mobile/copyright/wikitech-wordmark.svg" style="width: 8.75em; height: 1.6875em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikitech [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikitech" aria-label="Search Wikitech" autocapitalize="sentences" title="Search Wikitech [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-uls" class="mw-list-item active user-links-collapsible-item"><a data-mw="interface" href="#" class="uls-trigger cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>English</span></a> </li> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=wikitech.wikimedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Portal%3AToolforge%2FAdmin" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out user-links-collapsible-item" title="More options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=wikitech.wikimedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Portal%3AToolforge%2FAdmin" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"><div id="mw-dismissablenotice-anonplace"></div><script>(function(){var node=document.getElementById("mw-dismissablenotice-anonplace");if(node){node.outerHTML="\u003Cdiv class=\"mw-dismissable-notice\"\u003E\u003Cdiv class=\"mw-dismissable-notice-close\"\u003E[\u003Ca tabindex=\"0\" role=\"button\"\u003Edismiss\u003C/a\u003E]\u003C/div\u003E\u003Cdiv class=\"mw-dismissable-notice-body\"\u003E\u003C!-- CentralNotice --\u003E\u003Cdiv id=\"localNotice\" data-nosnippet=\"\"\u003E\u003Cdiv class=\"sitenotice\" lang=\"en\" dir=\"ltr\"\u003E\u003Ctable style=\"width: 75%; background-color: var(--background-color-warning-subtle, #fdf2d5); border: var(--border-subtle, 1px solid #987027); color: var(--color-base, #202122); border-radius: 10px; padding: 5px; margin: 0 auto;\"\u003E\n\u003Ctbody\u003E\u003Ctr\u003E\n\u003Ctd style=\"width:40px; height:40px; text-align:center; vertical-align:middle; padding: 2px;\"\u003E\u003Cspan typeof=\"mw:File\"\u003E\u003Ca href=\"/wiki/File:OOjs_UI_icon_alert-warning.svg\" class=\"mw-file-description\"\u003E\u003Cimg src=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/30px-OOjs_UI_icon_alert-warning.svg.png\" decoding=\"async\" width=\"30\" height=\"30\" class=\"mw-file-element\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/45px-OOjs_UI_icon_alert-warning.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/60px-OOjs_UI_icon_alert-warning.svg.png 2x\" data-file-width=\"20\" data-file-height=\"20\" /\u003E\u003C/a\u003E\u003C/span\u003E\n\u003C/td\u003E\n\u003Ctd style=\"text-align:center; vertical-align: middle; padding: 4px; max-height: 60px;\"\u003E\u003Cb\u003EWe are migrating Wikitech to \u003Ca href=\"/wiki/News/2024_Migrating_Wikitech_Account_to_SUL\" title=\"News/2024 Migrating Wikitech Account to SUL\"\u003ESUL\u003C/a\u003E.\u003C/b\u003E Action may be required for your \u003Ca href=\"/wiki/News/2024_Migrating_Wikitech_Account_to_SUL#What_you_need_to_do\" title=\"News/2024 Migrating Wikitech Account to SUL\"\u003E account\u003C/a\u003E.\n\u003Cp\u003E\u003Cb\u003ETrouble logging in? Please visit \u003Ca href=\"https://phabricator.wikimedia.org/T376267\" class=\"extiw\" title=\"phab:T376267\"\u003ET376267\u003C/a\u003E\u003C/b\u003E\n\u003C/p\u003E\n\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E";}}());</script></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">Beginning</div> </a> </li> <li id="toc-Admin_permissions" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Admin_permissions"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Admin permissions</span> </div> </a> <ul id="toc-Admin_permissions-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Failover" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Failover"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Failover</span> </div> </a> <button aria-controls="toc-Failover-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Failover subsection</span> </button> <ul id="toc-Failover-sublist" class="vector-toc-list"> <li id="toc-WebProxy" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#WebProxy"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1</span> <span>WebProxy</span> </div> </a> <ul id="toc-WebProxy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Static_webserver" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Static_webserver"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.2</span> <span>Static webserver</span> </div> </a> <ul id="toc-Static_webserver-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Checker_service" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Checker_service"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.3</span> <span>Checker service</span> </div> </a> <ul id="toc-Checker_service-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Redis" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Redis"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.4</span> <span>Redis</span> </div> </a> <ul id="toc-Redis-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Prometheus" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Prometheus"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.5</span> <span>Prometheus</span> </div> </a> <ul id="toc-Prometheus-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Services" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Services"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.6</span> <span>Services</span> </div> </a> <ul id="toc-Services-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Command_orchestration" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Command_orchestration"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Command orchestration</span> </div> </a> <ul id="toc-Command_orchestration-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Administrative_tasks" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Administrative_tasks"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Administrative tasks</span> </div> </a> <button aria-controls="toc-Administrative_tasks-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Administrative tasks subsection</span> </button> <ul id="toc-Administrative_tasks-sublist" class="vector-toc-list"> <li id="toc-Logging_in_as_root" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Logging_in_as_root"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.1</span> <span>Logging in as root</span> </div> </a> <ul id="toc-Logging_in_as_root-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Disabling_all_ssh_logins_except_root" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Disabling_all_ssh_logins_except_root"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.2</span> <span>Disabling all ssh logins except root</span> </div> </a> <ul id="toc-Disabling_all_ssh_logins_except_root-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Complaints_of_bastion_being_slow" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Complaints_of_bastion_being_slow"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.3</span> <span>Complaints of bastion being slow</span> </div> </a> <ul id="toc-Complaints_of_bastion_being_slow-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Local_package_management" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Local_package_management"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4</span> <span>Local package management</span> </div> </a> <ul id="toc-Local_package_management-sublist" class="vector-toc-list"> <li id="toc-Local_package_policy" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Local_package_policy"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.1</span> <span>Local package policy</span> </div> </a> <ul id="toc-Local_package_policy-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Building_packages" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Building_packages"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.2</span> <span>Building packages</span> </div> </a> <ul id="toc-Building_packages-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deploy_new_misctools_package" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Deploy_new_misctools_package"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.3</span> <span>Deploy new misctools package</span> </div> </a> <ul id="toc-Deploy_new_misctools_package-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Testing/QA_for_a_new_tools-webservice_package" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Testing/QA_for_a_new_tools-webservice_package"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.4</span> <span>Testing/QA for a new tools-webservice package</span> </div> </a> <ul id="toc-Testing/QA_for_a_new_tools-webservice_package-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deploy_new_tools-webservice_package" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Deploy_new_tools-webservice_package"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.4.5</span> <span>Deploy new tools-webservice package</span> </div> </a> <ul id="toc-Deploy_new_tools-webservice_package-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Webserver_statistics" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Webserver_statistics"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.5</span> <span>Webserver statistics</span> </div> </a> <ul id="toc-Webserver_statistics-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Banning_an_IP_from_tool_labs" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Banning_an_IP_from_tool_labs"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.6</span> <span>Banning an IP from tool labs</span> </div> </a> <ul id="toc-Banning_an_IP_from_tool_labs-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deploying_the_main_web_page" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Deploying_the_main_web_page"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.7</span> <span>Deploying the main web page</span> </div> </a> <ul id="toc-Deploying_the_main_web_page-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Regenerate_replica.my.cnf" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Regenerate_replica.my.cnf"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.8</span> <span>Regenerate replica.my.cnf</span> </div> </a> <ul id="toc-Regenerate_replica.my.cnf-sublist" class="vector-toc-list"> <li id="toc-Debugging_bad_MariaDB_credentials" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Debugging_bad_MariaDB_credentials"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.8.1</span> <span>Debugging bad MariaDB credentials</span> </div> </a> <ul id="toc-Debugging_bad_MariaDB_credentials-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Regenerate_kubernetes_credentials_for_tools_(.kube/config)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Regenerate_kubernetes_credentials_for_tools_(.kube/config)"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.9</span> <span>Regenerate kubernetes credentials for tools (.kube/config)</span> </div> </a> <ul id="toc-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Adding_K8S_Components" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Adding_K8S_Components"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.10</span> <span>Adding K8S Components</span> </div> </a> <ul id="toc-Adding_K8S_Components-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Deleting_a_tool" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Deleting_a_tool"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.11</span> <span>Deleting a tool</span> </div> </a> <ul id="toc-Deleting_a_tool-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-SSL_certificates" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#SSL_certificates"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.12</span> <span>SSL certificates</span> </div> </a> <ul id="toc-SSL_certificates-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Granting_a_tool_write_access_to_Elasticsearch" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Granting_a_tool_write_access_to_Elasticsearch"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.13</span> <span>Granting a tool write access to Elasticsearch</span> </div> </a> <ul id="toc-Granting_a_tool_write_access_to_Elasticsearch-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Package_upgrades" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Package_upgrades"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.14</span> <span>Package upgrades</span> </div> </a> <ul id="toc-Package_upgrades-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)"> <div class="vector-toc-text"> <span class="vector-toc-numb">4.15</span> <span>Creating a new Docker image (e.g. for new versions of Node.js)</span> </div> </a> <ul id="toc-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Kubernetes" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Kubernetes"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Kubernetes</span> </div> </a> <ul id="toc-Kubernetes-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Build_service" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Build_service"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>Build service</span> </div> </a> <ul id="toc-Build_service-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Tools-mail_/_Exim" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Tools-mail_/_Exim"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>Tools-mail / Exim</span> </div> </a> <ul id="toc-Tools-mail_/_Exim-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Users_and_community" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Users_and_community"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>Users and community</span> </div> </a> <button aria-controls="toc-Users_and_community-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Users and community subsection</span> </button> <ul id="toc-Users_and_community-sublist" class="vector-toc-list"> <li id="toc-Project_membership_request_approval" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Project_membership_request_approval"> <div class="vector-toc-text"> <span class="vector-toc-numb">8.1</span> <span>Project membership request approval</span> </div> </a> <ul id="toc-Project_membership_request_approval-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Quota_management" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Quota_management"> <div class="vector-toc-text"> <span class="vector-toc-numb">8.2</span> <span>Quota management</span> </div> </a> <ul id="toc-Quota_management-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Other" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Other"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>Other</span> </div> </a> <button aria-controls="toc-Other-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Other subsection</span> </button> <ul id="toc-Other-sublist" class="vector-toc-list"> <li id="toc-How_do_Toolforge_web_services_actually_work?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#How_do_Toolforge_web_services_actually_work?"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.1</span> <span>How do Toolforge web services actually work?</span> </div> </a> <ul id="toc-How_do_Toolforge_web_services_actually_work?-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-What_makes_a_root/Giving_root_access" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#What_makes_a_root/Giving_root_access"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.2</span> <span>What makes a root/Giving root access</span> </div> </a> <ul id="toc-What_makes_a_root/Giving_root_access-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Servicegroup_log" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Servicegroup_log"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.3</span> <span>Servicegroup log</span> </div> </a> <ul id="toc-Servicegroup_log-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Useful_administrative_tools" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Useful_administrative_tools"> <div class="vector-toc-text"> <span class="vector-toc-numb">9.4</span> <span>Useful administrative tools</span> </div> </a> <ul id="toc-Useful_administrative_tools-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Brainstorming" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Brainstorming"> <div class="vector-toc-text"> <span class="vector-toc-numb">10</span> <span>Brainstorming</span> </div> </a> <ul id="toc-Brainstorming-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Sub_pages" class="vector-toc-list-item vector-toc-level-1"> <a class="vector-toc-link" href="#Sub_pages"> <div class="vector-toc-text"> <span class="vector-toc-numb">11</span> <span>Sub pages</span> </div> </a> <ul id="toc-Sub_pages-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" title="Table of Contents" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Portal:Toolforge/Admin</span></h1> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Portal:Toolforge/Admin" title="View the content page [c]" accesskey="c"><span>Page</span></a></li><li id="ca-talk" class="new vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Talk:Portal:Toolforge/Admin&action=edit&redlink=1" rel="discussion" class="new" title="Discussion about the content page (page does not exist) [t]" accesskey="t"><span>Discussion</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Portal:Toolforge/Admin"><span>Read</span></a></li><li id="ca-viewsource" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=edit" title="This page is protected.
You can view its source [e]" accesskey="e"><span>View source</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Portal:Toolforge/Admin"><span>Read</span></a></li><li id="ca-more-viewsource" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=edit"><span>View source</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Portal:Toolforge/Admin" title="A list of all wiki pages that link here [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Portal:Toolforge/Admin" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&oldid=2256172" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Portal%3AToolforge%2FAdmin&id=2256172&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FPortal%3AToolforge%2FAdmin"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FPortal%3AToolforge%2FAdmin"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-create_a_book" class="mw-list-item"><a href="/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Portal%3AToolforge%2FAdmin"><span>Create a book</span></a></li><li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Portal%3AToolforge%2FAdmin&action=show-download-screen"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Portal:Toolforge/Admin&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikitech</div> </div> <div id="contentSub"><div id="mw-content-subtitle"><div class="subpages">< <bdi dir="ltr"><a href="/wiki/Portal:Toolforge" title="Portal:Toolforge">Portal:Toolforge</a></bdi></div></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><style data-mw-deduplicate="TemplateStyles:r2241375">.mw-parser-output .tpl-navsidebar{max-width:22em;background:var(--background-color-base,#fff);color:var(--color-base,#202122);border:1px solid var(--border-color-base,#a2a9b1);float:right;clear:right;margin:.5em 0 1em 1em}.mw-parser-output .tpl-navsidebar-floatright{float:right;clear:right;margin:.5em 0 1em 1em}.mw-parser-output .tpl-navsidebar-floatleft{float:left;clear:left;margin:.5em 1em 1em 0}.mw-parser-output .tpl-navsidebar-floatnone{float:none;clear:both;margin:.5em 0}.mw-parser-output .tpl-navsidebar-topimage{margin:0 0 16px 0}.mw-parser-output .tpl-navsidebar-title{margin:8px 16px;border-bottom:3px solid var(--border-color-muted,#eaecf0);font-size:20px;text-align:center}.mw-parser-output .tpl-navsidebar-image{margin:0 0 8px}.mw-parser-output .tpl-navsidebar-content{margin:0 0 16px 0;padding:0 8px}.mw-parser-output .tpl-navsidebar-heading{margin:8px 0;font-weight:bold}.mw-parser-output .tpl-navsidebar-foot{padding:0 8px;margin:0;text-align:right;font-size:smaller}@media not (min-width:720px){.mw-parser-output .tpl-navsidebar{float:none;clear:both;margin:.5em 0;max-width:none}}</style><div role="navigation" class="navigation-not-searchable tpl-navsidebar" style=""><p class="tpl-navsidebar-title"><a class="mw-selflink selflink">Toolforge Admin</a></p><div class="tpl-navsidebar-image"><figure class="mw-halign-center" typeof="mw:File"><a href="/wiki/File:Toolforge_logo.svg" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Toolforge_logo.svg/50px-Toolforge_logo.svg.png" decoding="async" width="50" height="50" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Toolforge_logo.svg/75px-Toolforge_logo.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Toolforge_logo.svg/100px-Toolforge_logo.svg.png 2x" data-file-width="180" data-file-height="180"/></a><figcaption></figcaption></figure></div><div class="tpl-navsidebar-contents"><div class="tpl-navsidebar-content"> <div style="font-size: 85%;"> <div class="mw-inputbox-centered" style=""><form name="searchbox" class="searchbox mw-inputbox-form-inline" action="/wiki/Special:Search"><div class="cdx-text-input"><input class="mw-searchInput searchboxInput cdx-text-input__input" name="search" placeholder="Search Toolforge Admin Documentation" size="35" dir="ltr"/></div><input type="hidden" value=""Portal:Tool"" name="prefix"/> <input type="submit" name="fulltext" value="Search" class="cdx-button"/><input type="hidden" value="Search" name="fulltext"/></form></div> </div> </div><div class="tpl-navsidebar-content"> <ul><li><a href="/wiki/Help:Toolforge" title="Help:Toolforge">Toolforge user docs</a></li> <li><a class="mw-selflink selflink">Toolforge admin docs</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/System_Overview" title="Portal:Toolforge/Admin/System Overview">System overview</a></li> <li><a href="/wiki/Wikimedia_Cloud_Services_team/Ownership" title="Wikimedia Cloud Services team/Ownership">Reference: all WMCS-owned services</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Admin tasks and tools</p><p class="mw-empty-elt"> </p><ul><li><a class="mw-selflink-fragment" href="#Administrative_tasks">List of admin tasks</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a></li> <li><a class="external text" href="https://tool-db-usage.toolforge.org/">ToolsDB</a></li> <li><a class="external text" href="https://openstack-browser.toolforge.org/">OpenStack Browser</a></li> <li><a class="external text" href="https://toolhub.wikimedia.org/lists/192">Infrastructure tools</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading"><a class="mw-selflink-fragment" href="#Failover">Failover</a></p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Toolschecker" title="Portal:Toolforge/Admin/Toolschecker"> Toolschecker</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Redis" title="Portal:Toolforge/Admin/Redis"> Redis</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Prometheus#Failover" title="Portal:Toolforge/Admin/Prometheus"> Prometheus</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading"><a href="/wiki/Portal:Toolforge/Admin/Kubernetes" title="Portal:Toolforge/Admin/Kubernetes">Kubernetes</a></p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Certificates" title="Portal:Toolforge/Admin/Kubernetes/Certificates">Certificates</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Deploying" class="mw-redirect" title="Portal:Toolforge/Admin/Kubernetes/Deploying">Deploying</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Docker-registry" title="Portal:Toolforge/Admin/Kubernetes/Docker-registry">Docker-registry</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Jobs_framework" class="mw-redirect" title="Portal:Toolforge/Admin/Kubernetes/Jobs framework">Jobs framework</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Labels" title="Portal:Toolforge/Admin/Kubernetes/Labels">Labels</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Networking_and_ingress" title="Portal:Toolforge/Admin/Kubernetes/Networking and ingress">Networking and ingress</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Pod_tracing" title="Portal:Toolforge/Admin/Kubernetes/Pod tracing">Pod tracing</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Kubernetes services</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Jobs_framework" title="Portal:Toolforge/Admin/Jobs framework">Jobs framework</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Build_Service" title="Portal:Toolforge/Admin/Build Service">Build service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Envvars_Service" title="Portal:Toolforge/Admin/Envvars Service">Envvars service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Harbor" title="Portal:Toolforge/Admin/Harbor">Harbor</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Packaging</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">.deb packages</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Runbooks and emergency guides</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Runbooks" title="Portal:Toolforge/Admin/Runbooks">Portal:Toolforge/Admin/Runbooks</a></li> <li><a href="/wiki/Wikimedia_Cloud_Services_team/Incident_Response_Process" class="mw-redirect" title="Wikimedia Cloud Services team/Incident Response Process"> Incident response process</a></li></ul> </div><div class="tpl-navsidebar-content"> <p class="tpl-navsidebar-heading">Other</p><p class="mw-empty-elt"> </p><ul><li><a href="/wiki/Portal:Toolforge/Admin/Exim" title="Portal:Toolforge/Admin/Exim"> Exim</a> and <a href="/wiki/Portal:Cloud_VPS/Admin/Email#Operations" title="Portal:Cloud VPS/Admin/Email"> Email operations</a></li></ul> </div></div><p class="tpl-navsidebar-foot">[<span class="noprint plainlinks"><a class="external text" href="https://wikitech.wikimedia.org/w/index.php?title=Template:Toolforge_nav_admin&action=edit"><span title="Edit this template">edit</span></a></span>]</p></div> <p><br/> Documentation of backend components and <b>admin procedures for Toolforge</b>. See <a href="/wiki/Help:Toolforge" title="Help:Toolforge">Help:Toolforge</a> for user facing documentation about actually using Toolforge to run your bots and webservices. </p> <meta property="mw:PageProp/toc"/> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Admin_permissions" data-mw-thread-id="h-Admin_permissions"><span data-mw-comment-start="" id="h-Admin_permissions"></span>Admin permissions<span data-mw-comment-end="h-Admin_permissions"></span></h2></div> <p>Performing admin procedures requires having admin permissions on Toolforge. There is not a single "admin" flag, but a set of interrelated permissions you can be granted. These are described in detail in the page <a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a>. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Failover" data-mw-thread-id="h-Failover"><span data-mw-comment-start="" id="h-Failover"></span>Failover<span data-mw-comment-end="h-Failover"></span></h2></div> <p>Tools should be able to survive the failure of any one virt* node. Some items may need manual failover </p> <div class="mw-heading mw-heading3"><h3 id="WebProxy" data-mw-thread-id="h-WebProxy-Failover"><span data-mw-comment-start="" id="h-WebProxy-Failover"></span>WebProxy<span data-mw-comment-end="h-WebProxy-Failover"></span></h3></div> <style data-mw-deduplicate="TemplateStyles:r2199615">.mw-parser-output .tracked{border:1px solid var(--border-color-base,#a2a9b1);border-radius:.5em;background-color:var(--background-color-interactive,#eaecf0);background-image:linear-gradient(var(--background-color-interactive,#eaecf0),var(--background-color-interactive-subtle,#f8f9fa));color:var(--color-base,#202122);font-size:85%;text-align:center;margin-bottom:1em;width:12em;padding:0.5em}.mw-parser-output .tracked-status{color:var(--color-emphasized,#000);font-weight:bold;text-transform:uppercase}.mw-parser-output .tracked-status--resolved{color:var(--color-success,#14866d)}.mw-parser-output .tracked-status--critical{color:var(--color-error,#d73333);font-size:1.5em}</style> <div style="float:right; clear:right; margin-left:1em;" class="plainlinks tracked mw-trackedTemplate">Tracked in <a href="https://phabricator.wikimedia.org/" class="extiw" title="phabricator:">Phabricator</a><br/><b><a href="https://phabricator.wikimedia.org/T283948" class="extiw" title="phabricator:T283948"><span class="trakfab-T283948">Task T283948</span></a></b></div> <p>The front web proxy is now a stateless web service. </p><p>There are two <code>tools-proxy-N</code> VMs in the <code>tools</code> project, which previously ran <a href="/wiki/Obsolete:Portal:Toolforge/Admin/Dynamicproxy" title="Obsolete:Portal:Toolforge/Admin/Dynamicproxy">Dynamicproxy</a> and nowadays just proxy everything to the <a href="/wiki/Portal:Toolforge/Admin/Kubernetes/New_cluster#front_proxy_(haproxy)" title="Portal:Toolforge/Admin/Kubernetes/New cluster">K8s HAProxies</a>. </p><p>The only meaningful thing that currently happens on them is the toolviews counting based on the access logs. Otherwise we could remove those nodes and just point to HAProxy. </p><p>In case one VM is not working correctly, we can failover from one VM to the other, which can be done by manually reassigning the floating IP in Horizon or from the OpenStack CLI. </p> <style data-mw-deduplicate="TemplateStyles:r2211903">.mw-parser-output .note{background-position:left 7px top 50%;padding:0.5em 0.5em 0.5em 40px;margin:0.5em 0;overflow:hidden;background-color:#f8f9fa;color:#333;background-repeat:no-repeat;border:1px solid #ddd}.mw-parser-output .note-inline{display:inline-block;vertical-align:middle}.mw-parser-output .note-info{background-color:#eaf3ff;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg");background-size:25px;border-color:#a3caff;padding-left:40px;min-height:25px}.mw-parser-output .note-reminder{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/a/a8/OOjs_UI_icon_lightbulb-yellow.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-warn{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/3/3b/OOjs_UI_icon_alert-warning.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-error{background-color:#fee7e6;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/b/bf/OOjs_UI_icon_notice-destructive.svg");background-size:25px;border-color:#c33;min-height:25px}@media screen{html.skin-theme-clientpref-night .mw-parser-output .note{background-color:transparent;color:inherit}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .note{background-color:transparent;color:inherit}}</style><div role="note" class="note note-info">This is a different proxy from the <a href="/wiki/Portal:Cloud_VPS/Admin/Web_proxy" title="Portal:Cloud VPS/Admin/Web proxy">Cloud VPS Web Proxy</a>.</div> <div class="mw-heading mw-heading3"><h3 id="Static_webserver" data-mw-thread-id="h-Static_webserver-Failover"><span data-mw-comment-start="" id="h-Static_webserver-Failover"></span>Static webserver<span data-mw-comment-end="h-Static_webserver-Failover"></span></h3></div> <p>This is a stateless simple nginx http server. Simply switch the floating IP from tools-static-10 to tools-static-11 (or vice versa) to switch over. Recovery is also equally trivial - just bring the machine back up and make sure puppet is ok. </p> <div class="mw-heading mw-heading3"><h3 id="Checker_service" data-mw-thread-id="h-Checker_service-Failover"><span data-mw-comment-start="" id="h-Checker_service-Failover"></span>Checker service<span data-mw-comment-end="h-Checker_service-Failover"></span></h3></div> <p>This is the service that Icinga hits to check status of several services. It's totally stateless. </p><p>See <a href="/wiki/Portal:Toolforge/Admin/Toolschecker" title="Portal:Toolforge/Admin/Toolschecker">Portal:Toolforge/Admin/Toolschecker</a> </p> <div class="mw-heading mw-heading3"><h3 id="Redis" data-mw-thread-id="h-Redis-Failover"><span data-mw-comment-start="" id="h-Redis-Failover"></span>Redis<span data-mw-comment-end="h-Redis-Failover"></span></h3></div> <p>Redis uses <a href="/wiki/Portal:Toolforge/Admin/Redis" title="Portal:Toolforge/Admin/Redis">Sentinel</a> to automatically fail over in case of a node failure. </p> <div class="mw-heading mw-heading3"><h3 id="Prometheus" data-mw-thread-id="h-Prometheus-Failover"><span data-mw-comment-start="" id="h-Prometheus-Failover"></span>Prometheus<span data-mw-comment-end="h-Prometheus-Failover"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Prometheus#Failover" title="Portal:Toolforge/Admin/Prometheus">Portal:Toolforge/Admin/Prometheus#Failover</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Services" data-mw-thread-id="h-Services-Failover"><span data-mw-comment-start="" id="h-Services-Failover"></span>Services<span data-mw-comment-end="h-Services-Failover"></span></h3></div> <p>Service nodes run the Toolforge internal <b>aptly</b> service, to serve .deb packages as a repository for all the other nodes. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Command_orchestration" data-mw-thread-id="h-Command_orchestration"><span data-mw-comment-start="" id="h-Command_orchestration"></span>Command orchestration<span data-mw-comment-end="h-Command_orchestration"></span></h2></div> <p>Toolforge and Toolsbeta both have a local <a href="/wiki/Cumin" title="Cumin">cumin</a> server. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Administrative_tasks" data-mw-thread-id="h-Administrative_tasks"><span data-mw-comment-start="" id="h-Administrative_tasks"></span>Administrative tasks<span data-mw-comment-end="h-Administrative_tasks"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="Logging_in_as_root" data-mw-thread-id="h-Logging_in_as_root-Administrative_tasks"><span data-mw-comment-start="" id="h-Logging_in_as_root-Administrative_tasks"></span>Logging in as root<span data-mw-comment-end="h-Logging_in_as_root-Administrative_tasks"></span></h3></div> <p>For normal login root access see <a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a>. </p><p>In case the normal login does not work for example due to an LDAP failure, administrators can also directly log in as root. To prepare for that occasion, generate a separate key with <code>ssh-keygen</code>, add an entry to the <code>passwords::root::extra_keys</code> hash in Horizon's 'Project Puppet' section with your shell username as key and your public key as value and wait a Puppet cycle to have your key added to the <code>root</code> accounts. Add to your <code>~/.ssh/config</code>: </p> <pre># Use different identity for Tools root. Match host *.tools.eqiad1.wikimedia.cloud user root IdentityFile ~/.ssh/your_secret_root_key </pre> <p>The code that reads <code>passwords::root::extra_keys</code> is in <a class="external text" href="https://phabricator.wikimedia.org/diffusion/LPRI/browse/master/modules/passwords/manifests/init.pp">labs/private:modules/passwords/manifests/init.pp</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Disabling_all_ssh_logins_except_root" data-mw-thread-id="h-Disabling_all_ssh_logins_except_root-Administrative_tasks"><span data-mw-comment-start="" id="h-Disabling_all_ssh_logins_except_root-Administrative_tasks"></span>Disabling all ssh logins except root<span data-mw-comment-end="h-Disabling_all_ssh_logins_except_root-Administrative_tasks"></span></h3></div> <p>Useful for dealing with security critical situations. Just touch <code>/etc/nologin</code> and PAM will prevent any and all non-root logins. </p> <div class="mw-heading mw-heading3"><h3 id="Complaints_of_bastion_being_slow" data-mw-thread-id="h-Complaints_of_bastion_being_slow-Administrative_tasks"><span data-mw-comment-start="" id="h-Complaints_of_bastion_being_slow-Administrative_tasks"></span>Complaints of bastion being slow<span data-mw-comment-end="h-Complaints_of_bastion_being_slow-Administrative_tasks"></span></h3></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199615"> <div style="float:right; clear:right; margin-left:1em;" class="plainlinks tracked mw-trackedTemplate">Tracked in <a href="https://phabricator.wikimedia.org/" class="extiw" title="phabricator:">Phabricator</a><br/><b><a href="https://phabricator.wikimedia.org/T266300" class="extiw" title="phabricator:T266300"><span class="trakfab-T266300">Task T266300</span></a></b></div> <p>Users are increasingly noticing slowness on tools-login due to either CPU or IOPS exhaustion caused by people running processes there instead of on Kubernetes. Here are some tips for finding the processes in need of killing: </p> <ul><li>Look for IOPS hogs <ul><li><code class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><span class="gp">$ </span>iotop</code></li></ul></li> <li>Look for abnormal processes: <ul><li><code class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><span class="gp">$ </span>ps<span class="w"> </span>axo<span class="w"> </span>user:32,pid,cmd<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-Ev<span class="w"> </span><span class="s2">"^(</span><span class="nv">$USER</span><span class="s2">|root|daemon|_lldpd|messagebus|nagios|nslcd|ntp|prometheus|statd|syslog|Debian-exim|www-data)"</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-ivE<span class="w"> </span><span class="s1">'screen|tmux|-bash|mosh-server|sshd:|/bin/bash|/bin/zsh'</span></code></li> <li>If you see <code>pyb.py</code> kill with extreme prejudice.</li></ul></li> <li>If the rogue job is running as a tool, <code>!log</code> something like: <code>!log tools.$TOOL Killed $PROC process running on tools-bastion-NN. See https://wikitech.wikimedia.org/wiki/Help:Toolforge/Jobs_framework for instructions on running jobs on Kubernetes.</code></li></ul> <div class="mw-heading mw-heading3"><h3 id="Local_package_management" data-mw-thread-id="h-Local_package_management-Administrative_tasks"><span data-mw-comment-start="" id="h-Local_package_management-Administrative_tasks"></span>Local package management<span data-mw-comment-end="h-Local_package_management-Administrative_tasks"></span></h3></div> <p>Local packages are provided by an <code>aptly</code> repository on <code>tools-services-05</code>. </p><p>On <code>tools-services-05</code>, you can manipulate the package database by various commands; cf. <code>aptly(1)</code>. Afterwards, you need to publish the database to the file <code>Packages</code> by (for the <code>trusty-tools</code> repository) <code>aptly publish --skip-signing update trusty-tools</code>. To use the packages on the clients you need to wait 30 minutes again or run <code>apt-get update</code>. In general, you should never just delete packages, but move them to <code>~tools.admin/archived-packages</code>. </p><p>You can always see where a package is (would be) coming from with <code>apt-cache showpkg $package</code>. </p> <div class="mw-heading mw-heading4"><h4 id="Local_package_policy" data-mw-thread-id="h-Local_package_policy-Local_package_management"><span data-mw-comment-start="" id="h-Local_package_policy-Local_package_management"></span>Local package policy<span data-mw-comment-end="h-Local_package_policy-Local_package_management"></span></h4></div> <p><b>Package repositories</b> </p> <ul><li>We only install packages from trustworthy repositories. <ul><li>OK are <ul><li>The official Debian and Ubuntu repositories, and</li> <li>Self-built packages (apt.wikimedia.org and aptly)</li></ul></li> <li>Not OK are: <ul><li>PPAs</li> <li>other 3rd party repositories</li></ul></li></ul></li></ul> <p><i>Packagers effectively get root on our systems, as they could add a rootkit to the package, or upload an unsafe sshd version, and apt-get will happily install it</i> </p><p>Hardness clause: in extraordinary cases, and for 'grandfathered in' packages, we can deviate from this policy, as long as security and maintainability are kept in mind. </p><p><b>apt.wikimedia.org</b> </p><p>We assume that whatever is good for production is also OK for Toolforge. </p><p><b>aptly</b> </p><p>We manage the aptly repository ourselves. </p> <ul><li>Packages in aptly need to be built by Toolforge admins <ul><li>we cannot import .deb files from untrusted 3rd party sources</li></ul></li></ul> <ul><li>Package source files need to come from a trusted source <ul><li>a source file from a trusted source (i.e. backports), or</li> <li>we build the debian source files ourselves</li> <li>we cannot build .dcs files from untrusted 3rd party sources</li></ul></li></ul> <ul><li>Packages need to be easy to update and build <ul><li>cowbuilder/pdebuild OK</li> <li>fpm is OK</li> <li>See <a href="/wiki/Nova_Resource:Tools/Admin/Deploy_new_jobutils_package" class="mw-redirect" title="Nova Resource:Tools/Admin/Deploy new jobutils package">Deploy new jobutils package</a> for an example walk through of building and adding packages to aptly.</li></ul></li></ul> <ul><li>We only package if strictly necessary <ul><li>infrastructure packages</li> <li>packages that should be available for effective development (e.g. composer or sbt)</li> <li>not: python-*, lib*-perl, ..., which should just be installed with the available platform-specific package managers</li></ul></li></ul> <ul><li>For each package, it should be clear who is responsible for keeping it up to date <ul><li>for infrastructure packages, this should be one of the paid staffers</li></ul></li></ul> <p>A list of locally maintained packages can be found under <a href="/wiki/Nova_Resource:Tools/Admin/local_packages" class="mw-redirect" title="Nova Resource:Tools/Admin/local packages">/local packages</a>. </p> <div class="mw-heading mw-heading4"><h4 id="Building_packages" data-mw-thread-id="h-Building_packages-Local_package_management"><span data-mw-comment-start="" id="h-Building_packages-Local_package_management"></span>Building packages<span data-mw-comment-end="h-Building_packages-Local_package_management"></span></h4></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2211903"><div role="note" class="note note-info"> moved to <a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Portal:Toolforge/Admin/Packaging</a></div> <div class="mw-heading mw-heading4"><h4 id="Deploy_new_misctools_package" data-mw-thread-id="h-Deploy_new_misctools_package-Local_package_management"><span data-mw-comment-start="" id="h-Deploy_new_misctools_package-Local_package_management"></span>Deploy new misctools package<span data-mw-comment-end="h-Deploy_new_misctools_package-Local_package_management"></span></h4></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2211903"><div role="note" class="note note-info"> moved to <a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Portal:Toolforge/Admin/Packaging</a> </div> <div class="mw-heading mw-heading4"><h4 id="Testing/QA_for_a_new_tools-webservice_package" data-mw-thread-id="h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management"><span id="Testing.2FQA_for_a_new_tools-webservice_package"></span><span data-mw-comment-start="" id="h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management"></span>Testing/QA for a new tools-webservice package<span data-mw-comment-end="h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management"></span></h4></div> <p>See also <a class="external text" href="https://gerrit.wikimedia.org/r/plugins/gitiles/operations/software/tools-webservice/+/refs/heads/master/README">tools-webservice source tree README</a>. </p><p>There is a simple flask app in toolsbeta using the tool <code>test</code> that is set up to be deployed via webservice on Kubernetes. </p><p>After running <code>become test</code>, you can go to the <code>qa/tools-webservice</code> directory. This is checked out via anonymous https, and is suitable for checking out a patch you are reviewing. There is an untracked file in there that is useful here, usually. The webservicefile at the route is just a copy of the one in the <code>scripts</code> folder in the repo. The only difference is: </p> <div class="mw-highlight mw-highlight-lang-diff mw-content-ltr" dir="ltr"><pre><span></span><span class="gu">9d8</span> <span class="gd">< sys.path.insert(0, '')</span> </pre></div> <p>That exchanges the distribution installed package in the python path for the local directory, so if you run <code>./webservice $somecommand</code> it will run what is in your local folder rather than what is in <code>/usr/lib/python3/dist-packages/</code>. If you are testing changes made directly to <code>scripts/webservice</code> in the repo, you will likely need to copy that over the file and add <code>sys.path.insert(0, "")</code> after the import sys line. </p><p>If there is no <code>import sys</code> line in this version of the code, add one! This should let you bang on your new version without having to mess with packaging, yet. </p> <div class="mw-heading mw-heading4"><h4 id="Deploy_new_tools-webservice_package" data-mw-thread-id="h-Deploy_new_tools-webservice_package-Local_package_management"><span data-mw-comment-start="" id="h-Deploy_new_tools-webservice_package-Local_package_management"></span>Deploy new tools-webservice package<span data-mw-comment-end="h-Deploy_new_tools-webservice_package-Local_package_management"></span></h4></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2211903"><div role="note" class="note note-info"> moved to <a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Portal:Toolforge/Admin/Packaging</a> </div> <div class="mw-heading mw-heading3"><h3 id="Webserver_statistics" data-mw-thread-id="h-Webserver_statistics-Administrative_tasks"><span data-mw-comment-start="" id="h-Webserver_statistics-Administrative_tasks"></span>Webserver statistics<span data-mw-comment-end="h-Webserver_statistics-Administrative_tasks"></span></h3></div> <p>To get a look at webserver statistics, <a rel="nofollow" class="external text" href="//goaccess.io">goaccess</a> is installed on the webproxies. Usage: </p> <pre>goaccess --date-format="%d/%b/%Y" --log-format='%h - - [%d:%t %^] "%r" %s %b "%R" "%u"' -q -f/var/log/nginx/access.log</pre> <p>Interactive key bindings are documented on <a rel="nofollow" class="external text" href="http://goaccess.io/man#interactive-keys">the man page</a>. HTML output is supported by piping to a file. Note that nginx logs are rotated (twice?) daily, so there is only very recent data available. </p> <div class="mw-heading mw-heading3"><h3 id="Banning_an_IP_from_tool_labs" data-mw-thread-id="h-Banning_an_IP_from_tool_labs-Administrative_tasks"><span data-mw-comment-start="" id="h-Banning_an_IP_from_tool_labs-Administrative_tasks"></span>Banning an IP from tool labs<span data-mw-comment-end="h-Banning_an_IP_from_tool_labs-Administrative_tasks"></span></h3></div> <p>On <a href="/w/index.php?title=Hiera:Tools&action=edit&redlink=1" class="new" title="Hiera:Tools (page does not exist)">Hiera:Tools</a>, add the IP to the list of dynamicproxy::banned_ips, then force a puppet run on the webproxies. Add a note to <a href="/wiki/Help:Toolforge/Banned" title="Help:Toolforge/Banned">Help:Toolforge/Banned</a> explaining why. The user will get a message like <a class="external autonumber" href="https://toolforge.org/.error/banned.html">[1]</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Deploying_the_main_web_page" data-mw-thread-id="h-Deploying_the_main_web_page-Administrative_tasks"><span data-mw-comment-start="" id="h-Deploying_the_main_web_page-Administrative_tasks"></span>Deploying the main web page<span data-mw-comment-end="h-Deploying_the_main_web_page-Administrative_tasks"></span></h3></div><p> This website (plus the 403/500/503 error pages) are hosted under <code>tools.admin</code>. To deploy,</p><div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>become<span class="w"> </span>admin <span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span>tool-admin-web <span class="gp">$ </span>git<span class="w"> </span>pull </pre></div> <div class="mw-heading mw-heading3"><h3 id="Regenerate_replica.my.cnf" data-mw-thread-id="h-Regenerate_replica.my.cnf-Administrative_tasks"><span data-mw-comment-start="" id="h-Regenerate_replica.my.cnf-Administrative_tasks"></span>Regenerate replica.my.cnf<span data-mw-comment-end="h-Regenerate_replica.my.cnf-Administrative_tasks"></span></h3></div> <style data-mw-deduplicate="TemplateStyles:r2199624">.mw-parser-output .mw-tpl-rellink{font-style:italic;padding-bottom:0.4em;padding-left:0;margin-bottom:0.4em;color:#555;border-bottom:1px solid var(--border-color-base,#a2a9b1)}</style> <div class="rellink mw-tpl-rellink">See also: <a href="/wiki/Portal:Data_Services/Admin/Wiki_Replicas#Account_management_(maintain-dbusers)" title="Portal:Data Services/Admin/Wiki Replicas">Portal:Data Services/Admin/Wiki Replicas#Account management (maintain-dbusers)</a></div> <p>This requires access to the cloudcontrol host <a rel="nofollow" class="external text" href="https://codesearch.wmcloud.org/search/?q=maintain_dbusers_primary&files=&excludeFiles=&repos=">which is running maintain-dbusers</a>, and can be done as follows: </p> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>cloudcontrolXXXX.eqiad.wmnet <span class="gp">$ </span>sudo<span class="w"> </span>/usr/local/sbin/maintain-dbusers<span class="w"> </span>delete<span class="w"> </span>tools.<span class="si">${</span><span class="nv">NAME</span><span class="si">}</span><span class="w"> </span>--account-type<span class="o">=</span>tool <span class="go">:# or</span> <span class="gp">$ </span>sudo<span class="w"> </span>/usr/local/sbin/maintain-dbusers<span class="w"> </span>delete<span class="w"> </span><span class="si">${</span><span class="nv">USERNAME</span><span class="si">}</span><span class="w"> </span>--account-type<span class="o">=</span>user </pre></div> <p>Once the account has been deleted, the maintain-dbusers service will automatically recreate the user account. </p> <div class="mw-heading mw-heading4"><h4 id="Debugging_bad_MariaDB_credentials" data-mw-thread-id="h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf"><span data-mw-comment-start="" id="h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf"></span>Debugging bad MariaDB credentials<span data-mw-comment-end="h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf"></span></h4></div> <p><span id="Debugging_bad_mysql_credentials"></span> Sometimes things go wrong and a user's <code>replica.my.cnf</code> credentials don't propigate everywhere. You can check the status on various servers to try and narrow down what went wrong. </p><p>The database credentials needed are in <code>/etc/dbusers.yaml</code> on the cloudcontrol host running maintain-dbusers. </p> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>cloudcontrolXXXX.eqiad.wmnet <span class="gp">$ </span>sudo<span class="w"> </span>cat<span class="w"> </span>/etc/dbusers.yaml <span class="go">:# look for the accounts-backend['password'] for the m5-master connections (user: labsdbaccounts)</span> <span class="go">:# look for the labsdbs['password'] for the other connections (user: labsdbadmin)</span> <span class="gp">$ </span><span class="nv">CHECK_UID</span><span class="o">=</span>u12345<span class="w"> </span><span class="c1"># User id to check for</span> <span class="go">:# Check if the user is in our meta datastore</span> <span class="gp">$ </span>mariadb<span class="w"> </span>-h<span class="w"> </span>m5-master.eqiad.wmnet<span class="w"> </span>-u<span class="w"> </span>labsdbaccounts<span class="w"> </span>-p<span class="w"> </span>-e<span class="w"> </span><span class="s2">"USE labsdbaccounts; SELECT * FROM account WHERE mysql_username='</span><span class="si">${</span><span class="nv">CHECK_UID</span><span class="si">}</span><span class="s2">'\G"</span> <span class="go">:# Check if all the accounts are created in the labsdb boxes from meta datastore.</span> <span class="gp">$ </span><span class="nv">ACCT_ID</span><span class="o">=</span>....<span class="w"> </span><span class="c1"># Account_id is foreign key (id from account table)</span> <span class="gp">$ </span>mariadb<span class="w"> </span>-h<span class="w"> </span>m5-master.eqiad.wmnet<span class="w"> </span>-u<span class="w"> </span>labsdbaccounts<span class="w"> </span>-p<span class="w"> </span>-e<span class="w"> </span><span class="s2">"USE labsdbaccounts; SELECT * FROM labsdbaccounts.account_host WHERE account_id=</span><span class="si">${</span><span class="nv">ACCT_ID</span><span class="si">}</span><span class="s2">\G"</span> <span class="go">:# Check the actual labsdbs if needed</span> <span class="gp">$ </span>mariadb<span class="w"> </span>-h<span class="w"> </span>clouddbXXXX.eqiad.wmnet<span class="w"> </span>-u<span class="w"> </span>labsdbadmin<span class="w"> </span>-p<span class="w"> </span>-e<span class="w"> </span><span class="s1">'SELECT User, Password from mysql.user where User like "${CHECK_UID}";'</span> <span class="go">:# Resynchronize account state on the replicas by finding missing GRANTS on each db server</span> <span class="gp">$ </span>sudo<span class="w"> </span>maintain-dbusers<span class="w"> </span>harvest-replicas </pre></div> <p>See <a href="https://phabricator.wikimedia.org/T183644" class="extiw" title="phab:T183644">phab:T183644</a> for an example of fixing automatic credential creation caused when a old LDAP user becomes a Toolforge member and has an untracked user account on toolsdb. </p> <div class="mw-heading mw-heading3"><h3 id="Regenerate_kubernetes_credentials_for_tools_(.kube/config)" data-mw-thread-id="h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks"><span id="Regenerate_kubernetes_credentials_for_tools_.28.kube.2Fconfig.29"></span><span data-mw-comment-start="" id="h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks"></span>Regenerate kubernetes credentials for tools (.kube/config)<span data-mw-comment-end="h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks"></span></h3></div> <p>With admin credentials (root on a control plane node will do), run <code>kubectl -n tool-<toolname> delete cm maintain-kubeusers-<toolname></code>; it should get regenerated within minutes. </p> <div class="mw-heading mw-heading3"><h3 id="Adding_K8S_Components" data-mw-thread-id="h-Adding_K8S_Components-Administrative_tasks"><span data-mw-comment-start="" id="h-Adding_K8S_Components-Administrative_tasks"></span>Adding K8S Components<span data-mw-comment-end="h-Adding_K8S_Components-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes#Building_new_nodes" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes#Building_new_nodes</a> </p> <div class="mw-heading mw-heading3"><h3 id="Deleting_a_tool" data-mw-thread-id="h-Deleting_a_tool-Administrative_tasks"><span data-mw-comment-start="" id="h-Deleting_a_tool-Administrative_tasks"></span>Deleting a tool<span data-mw-comment-end="h-Deleting_a_tool-Administrative_tasks"></span></h3></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2199615"> <div style="float:right; clear:right; margin-left:1em;" class="plainlinks tracked mw-trackedTemplate">Tracked in <a href="https://phabricator.wikimedia.org/" class="extiw" title="phabricator:">Phabricator</a><br/><b><a href="https://phabricator.wikimedia.org/T170355" class="extiw" title="phabricator:T170355"><span class="trakfab-T170355">Task T170355</span></a></b> <span class="tracked-status tracked-status--resolved">Resolved</span></div> <p>For batch or CLI deletion of tools, use the 'mark_tool' command on a cloudcontrol node: </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Tool_disable_process.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/labs/thumb/a/a5/Tool_disable_process.png/220px-Tool_disable_process.png" decoding="async" width="220" height="186" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/labs/thumb/a/a5/Tool_disable_process.png/330px-Tool_disable_process.png 1.5x, //upload.wikimedia.org/wikipedia/labs/thumb/a/a5/Tool_disable_process.png/440px-Tool_disable_process.png 2x" data-file-width="1671" data-file-height="1411"/></a><figcaption>The awful truth about tool deletion</figcaption></figure> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">andrew@cloudcontrol1003:~$ </span>sudo<span class="w"> </span>mark_tool <span class="go">usage: mark_tool [-h] [--ldap-user LDAP_USER] [--ldap-password LDAP_PASSWORD]</span> <span class="go"> [--ldap-base-dn LDAP_BASE_DN] [--project PROJECT] [--disable]</span> <span class="go"> [--delete] [--enable]</span> <span class="go"> tool</span> <span class="go">mark_tool: error: the following arguments are required: tool</span> </pre></div> <p>Maintainers can mark their tools for deletion using the "Disable tool" button on the tool's detail page on <a class="external free" href="https://toolsadmin.wikimedia.org/">https://toolsadmin.wikimedia.org/</a>. In either case, the immediate effect of disabling a tool is to stop any running jobs, prevent users from logging in as that tool, and schedule archiving and deletion for 40 days in the future. </p> <figure class="mw-default-size" typeof="mw:File/Thumb"><a href="/wiki/File:Tool_restore_process.png" class="mw-file-description"><img src="//upload.wikimedia.org/wikipedia/labs/thumb/7/75/Tool_restore_process.png/220px-Tool_restore_process.png" decoding="async" width="220" height="122" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/labs/thumb/7/75/Tool_restore_process.png/330px-Tool_restore_process.png 1.5x, //upload.wikimedia.org/wikipedia/labs/thumb/7/75/Tool_restore_process.png/440px-Tool_restore_process.png 2x" data-file-width="1671" data-file-height="930"/></a><figcaption>A tool can be restored within 40 days of being disabled</figcaption></figure> <p>Tool archives are stored on the tools NFS server, currently <code>tools-nfs-2.tools.eqiad1.wikimedia.cloud</code>: </p> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">root@labstore1004:/srv/disable-tool# </span>ls<span class="w"> </span>-ltrah<span class="w"> </span>/srv/tools/archivedtools/ <span class="go">total 1.8G</span> <span class="go">drwxr-xr-x 5 root root 4.0K Jun 21 19:37 ..</span> <span class="go">-rw-r--r-- 1 root root 102K Jul 22 22:15 andrewtesttooltwo</span> <span class="go">-rw-r--r-- 1 root root 45 Oct 13 00:47 andrewtesttooltwo.tgz</span> <span class="go">-rw-r--r-- 1 root root 8.3M Oct 13 03:20 mediaplaycounts.tgz</span> <span class="go">-rw-r--r-- 1 root root 1.8G Oct 13 04:01 projanalysis.tgz</span> <span class="go">-rw-r--r-- 1 root root 1.3M Oct 13 21:05 reportsbot.tgz</span> <span class="go">drwxr-xr-x 2 root root 4.0K Oct 13 21:10 .</span> <span class="go">-rw-r--r-- 1 root root 719K Oct 13 21:10 wsm.tgz</span> <span class="go">-rw-r--r-- 1 root root 4.8K Oct 13 21:20 andrewtesttoolfour.tgz</span> </pre></div> <p>The actual deletion process is shockingly complicated. A tool will only be archived and deleted if all of the prior steps succeed, but disabling of a tool should be a sure thing. </p> <div class="mw-heading mw-heading3"><h3 id="SSL_certificates" data-mw-thread-id="h-SSL_certificates-Administrative_tasks"><span data-mw-comment-start="" id="h-SSL_certificates-Administrative_tasks"></span>SSL certificates<span data-mw-comment-end="h-SSL_certificates-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/SSL_certificates" title="Portal:Toolforge/Admin/SSL certificates">Portal:Toolforge/Admin/SSL_certificates</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Granting_a_tool_write_access_to_Elasticsearch" data-mw-thread-id="h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks"><span data-mw-comment-start="" id="h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks"></span>Granting a tool write access to Elasticsearch<span data-mw-comment-end="h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks"></span></h3></div> <ul><li>Generate a random password and the mkpassword crypt entry for it using the script <a href="https://phabricator.wikimedia.org/P4372" class="extiw" title="phab:P4372">new-es-password.sh</a>. (Must be run a host with the `mkpasswd` command installed. (The mkpasswd is part of the whois Debian package.)</li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>./new-es-password.sh<span class="w"> </span>tools.example <span class="go">tools.example elasticsearch.ini</span> <span class="go">----</span> <span class="go">[elasticsearch]</span> <span class="go">user=tools.example </span> <span class="go">password=A3rJqgFKxa/x4NlnIhmw2cXcV92it/Zv0Yt+a7yhxCw=</span> <span class="go">----</span> <span class="go">tools.example puppet master private (hieradata/labs/tools/common.yaml)</span> <span class="go">----</span> <span class="go">profile::toolforge::elasticsearch::haproxy::elastic_users:</span> <span class="go"> - name: 'tools.example'</span> <span class="go"> password: '$6$FYwP3wxT4K7O9EE$OA3P5972NWJVG/WUnD240sal34/dsNabbcawItevMYO9uoR.fJBrjSABex0EDW0wlkWHID1Tf4oJoiNvYFGmy/'</span> </pre></div> <ul><li>Add the private SHA512 hash to the <a href="/wiki/Portal:Toolforge/Nodes#misc_nodes" title="Portal:Toolforge/Nodes">tools puppetmaster</a>:</li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>tools-puppetserver-01.tools.eqiad1.wikimedia.cloud <span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span>/srv/git/labs/private <span class="gp">$ </span>sudo<span class="w"> </span>-i<span class="w"> </span>vim<span class="w"> </span>/var/lib/git/labs/private/hieradata/labs/tools/common.yaml <span class="go">... merge the hiera data with the existing key...</span> <span class="go">:wq</span> <span class="gp">$ </span>sudo<span class="w"> </span>git<span class="w"> </span>add<span class="w"> </span>hieradata/labs/tools/common.yaml <span class="gp">$ </span>sudo<span class="w"> </span>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">"[local] Elasticsearch credentials for </span><span class="nv">$TOOL</span><span class="s2">"</span> </pre></div> <ul><li>Force a puppet run on tools-elastic nodes using <a href="/wiki/Cumin" title="Cumin">Cumin</a></li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">cloudcumin1001.eqiad.wmnet:~$ </span>sudo<span class="w"> </span>cumin<span class="w"> </span><span class="s2">"O{project:tools name:.*elastic.*}"</span><span class="w"> </span><span class="s2">"run-puppet-agent"</span> </pre></div> <ul><li>Make the credentials available to the tool as <a href="/wiki/Help:Toolforge/Envvars_Service" class="mw-redirect" title="Help:Toolforge/Envvars Service">envvars</a>:</li></ul> <div class="mw-highlight mw-highlight-lang-shell-session mw-content-ltr" dir="ltr"><pre><span></span><span class="gp">$ </span>ssh<span class="w"> </span>dev.toolforge.org <span class="gp">$ </span>sudo<span class="w"> </span>-i<span class="w"> </span>become<span class="w"> </span>example-tool <span class="gp">$ </span>toolforge<span class="w"> </span>envvars<span class="w"> </span>create<span class="w"> </span>TOOL_ELASTICSEARCH_USER <span class="go">Enter the value of your envvar (prompt is hidden, hit Ctrl+C to abort): <insert user></span> <span class="gp">$ </span>toolforge<span class="w"> </span>envvars<span class="w"> </span>create<span class="w"> </span>TOOL_ELASTICSEARCH_PASSWORD <span class="go">Enter the value of your envvar (prompt is hidden, hit Ctrl+C to abort): <insert password></span> </pre></div> <p><b>Note:</b> An older procedure placed the credentials in <code>/data/project/$TOOL/.elasticsearch.ini</code> instead. </p> <ul><li>Resolve the ticket!</li></ul> <div class="mw-heading mw-heading3"><h3 id="Package_upgrades" data-mw-thread-id="h-Package_upgrades-Administrative_tasks"><span data-mw-comment-start="" id="h-Package_upgrades-Administrative_tasks"></span>Package upgrades<span data-mw-comment-end="h-Package_upgrades-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Cloud_VPS/Admin/Managing_package_upgrades" class="mw-redirect" title="Portal:Cloud VPS/Admin/Managing package upgrades">Managing package upgrades</a>. </p> <div class="mw-heading mw-heading3"><h3 id="Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)" data-mw-thread-id="h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks"><span id="Creating_a_new_Docker_image_.28e.g._for_new_versions_of_Node.js.29"></span><span data-mw-comment-start="" id="h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks"></span>Creating a new Docker image (e.g. for new versions of Node.js)<span data-mw-comment-end="h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes#Docker_Images" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes#Docker_Images</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Kubernetes" data-mw-thread-id="h-Kubernetes"><span data-mw-comment-start="" id="h-Kubernetes"></span>Kubernetes<span data-mw-comment-end="h-Kubernetes"></span></h2></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Build_service" data-mw-thread-id="h-Build_service"><span data-mw-comment-start="" id="h-Build_service"></span>Build service<span data-mw-comment-end="h-Build_service"></span></h2></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Build_Service" title="Portal:Toolforge/Admin/Build Service">Portal:Toolforge/Admin/Build_Service</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Tools-mail_/_Exim" data-mw-thread-id="h-Tools-mail_/_Exim"><span id="Tools-mail_.2F_Exim"></span><span data-mw-comment-start="" id="h-Tools-mail_/_Exim"></span>Tools-mail / Exim<span data-mw-comment-end="h-Tools-mail_/_Exim"></span></h2></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Exim" title="Portal:Toolforge/Admin/Exim">Portal:Toolforge/Admin/Exim</a> and <a href="/wiki/Portal:Cloud_VPS/Admin/Email#Operations" title="Portal:Cloud VPS/Admin/Email">Portal:Cloud_VPS/Admin/Email#Operations</a> </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Users_and_community" data-mw-thread-id="h-Users_and_community"><span data-mw-comment-start="" id="h-Users_and_community"></span>Users and community<span data-mw-comment-end="h-Users_and_community"></span></h2></div> <p>Some information about how to manage users and general community and their relationship with Toolforge. </p> <div class="mw-heading mw-heading3"><h3 id="Project_membership_request_approval" data-mw-thread-id="h-Project_membership_request_approval-Users_and_community"><span data-mw-comment-start="" id="h-Project_membership_request_approval-Users_and_community"></span>Project membership request approval<span data-mw-comment-end="h-Project_membership_request_approval-Users_and_community"></span></h3></div> <p>User access requests show up in <a class="external free" href="https://toolsadmin.wikimedia.org/tools/membership/">https://toolsadmin.wikimedia.org/tools/membership/</a> </p><p>Some guidelines for account approvals, based on <a href="https://phabricator.wikimedia.org/T128158#2132893" class="extiw" title="phab:T128158">advice from scfc</a>: </p> <ol><li>If the request contains any defamatory or abusive information as part of the username(s), reason, or comments → mark as <b>Declined</b> and check the "Suppress this request (hide from non-admin users)" checkbox. <ul><li>You should also block the user on Wikitech and consider contacting a <a href="https://meta.wikimedia.org/wiki/Stewards" class="extiw" title="meta:Stewards">Steward</a> for wider review of the SUL account.</li></ul></li> <li>If the user name "looks" like a bot or someone else who could not consent to the <a href="/wiki/Wikitech:Cloud_Services_Terms_of_use" title="Wikitech:Cloud Services Terms of use">Terms of use</a> and <a href="/wiki/Help:Toolforge/Rules" title="Help:Toolforge/Rules">Rules</a> → mark as <b>Declined</b>.</li> <li>Check the status of the associated <a href="https://meta.wikimedia.org/wiki/Help:Unified_login" class="extiw" title="meta:Help:Unified login">SUL account</a>. If the user is banned on one or more wikis → mark as <b>Declined</b>.</li> <li>If the stated purpose is "tangible" ("I want to move my bot x to Labs", "I want to build a web app that does y", etc.) → mark as <b>Approved</b>. <ul><li>If you know that someone else has been working on the same problem, add a message explaining who the user should contact or where they might find more information.</li></ul></li> <li>If the stated purpose is "abstract" ("research", "experimentation", etc.) and there is a hackathon ongoing or planned, the user has a non-throw-away mail address, the user has created a user page with coherent information about theirself or linked a SUL account of good standing, etc. → mark as <b>Approved</b>.</li> <li>Otherwise add a comment asking for clarification of their reason for use and mark as <b>Feedback needed</b>. The request is not really "denied", but more (indefinitely) "delayed".</li></ol> <p>Requests left in <b>Feedback needed</b> for more information for more than 30 days should usually be declined with a message like "Feel free to apply again later with more complete information." </p> <div class="mw-heading mw-heading3"><h3 id="Quota_management" data-mw-thread-id="h-Quota_management-Users_and_community"><span data-mw-comment-start="" id="h-Quota_management-Users_and_community"></span>Quota management<span data-mw-comment-end="h-Quota_management-Users_and_community"></span></h3></div> <p>Toolforge quotas are managed via maintain-kubeusers. </p> <ul><li>Have the user open a phabricator ticket, for the papertrail. See also <a href="/wiki/Help:Toolforge/Kubernetes#Quotas_and_Resources" title="Help:Toolforge/Kubernetes">Help:Toolforge/Kubernetes#Quotas_and_Resources</a></li></ul> <ul><li>Send a patch for maintain-kubeusers, have it reviewed and merged:</li></ul> <p><a class="external free" href="https://gitlab.wikimedia.org/repos/cloud/toolforge/toolforge-deploy/-/blob/main/components/maintain-kubeusers/values/tools.yaml">https://gitlab.wikimedia.org/repos/cloud/toolforge/toolforge-deploy/-/blob/main/components/maintain-kubeusers/values/tools.yaml</a> </p> <ul><li>Deploy in the cluster, using the deploy <a href="/wiki/Portal:Cloud_VPS/Admin/Cookbooks" title="Portal:Cloud VPS/Admin/Cookbooks">Portal:Cloud_VPS/Admin/Cookbooks</a></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Other" data-mw-thread-id="h-Other"><span data-mw-comment-start="" id="h-Other"></span>Other<span data-mw-comment-end="h-Other"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="How_do_Toolforge_web_services_actually_work?" data-mw-thread-id="h-How_do_Toolforge_web_services_actually_work?-Other"><span id="How_do_Toolforge_web_services_actually_work.3F"></span><span data-mw-comment-start="" id="h-How_do_Toolforge_web_services_actually_work?-Other"></span>How do Toolforge web services actually work?<span data-mw-comment-end="h-How_do_Toolforge_web_services_actually_work?-Other"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Kubernetes#Ingress" title="Portal:Toolforge/Admin/Kubernetes">Portal:Toolforge/Admin/Kubernetes#Ingress</a> </p> <div class="mw-heading mw-heading3"><h3 id="What_makes_a_root/Giving_root_access" data-mw-thread-id="h-What_makes_a_root/Giving_root_access-Other"><span id="What_makes_a_root.2FGiving_root_access"></span><span data-mw-comment-start="" id="h-What_makes_a_root/Giving_root_access-Other"></span>What makes a root/Giving root access<span data-mw-comment-end="h-What_makes_a_root/Giving_root_access-Other"></span></h3></div> <p>See <a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a> </p> <div class="mw-heading mw-heading3"><h3 id="Servicegroup_log" data-mw-thread-id="h-Servicegroup_log-Other"><span data-mw-comment-start="" id="h-Servicegroup_log-Other"></span>Servicegroup log<span data-mw-comment-end="h-Servicegroup_log-Other"></span></h3></div> <p>tools.admin runs <code>/data/project/admin/bin/toolhistory</code>, which provides an hourly snapshot of <code>ldaplist -l servicegroup</code> as git repository in <code>/data/project/admin/var/lib/git/servicegroups</code> </p> <div class="mw-heading mw-heading3"><h3 id="Useful_administrative_tools" data-mw-thread-id="h-Useful_administrative_tools-Other"><span data-mw-comment-start="" id="h-Useful_administrative_tools-Other"></span>Useful administrative tools<span data-mw-comment-end="h-Useful_administrative_tools-Other"></span></h3></div> <p>These tools offer useful information about Toolforge itself: </p> <ul><li><a href="https://iw.toolforge.org/tool-db-usage/" class="extiw" title="toolforge:tool-db-usage/">ToolsDB</a> - Statistics about tables owned by tools</li> <li><a href="https://iw.toolforge.org/k8s-stats" class="extiw" title="toolforge:k8s-stats">k8s-stats</a> - examine what our tools are doing</li> <li><a href="https://iw.toolforge.org/openstack-browser/project/tools" class="extiw" title="toolforge:openstack-browser/project/tools">OpenStack Browser</a> - examine projects, instances, web proxies, and Puppet config</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Brainstorming" data-mw-thread-id="h-Brainstorming"><span data-mw-comment-start="" id="h-Brainstorming"></span>Brainstorming<span data-mw-comment-end="h-Brainstorming"></span></h2></div> <ul><li><a href="/wiki/Portal:Toolforge/Admin/BotLicensing" class="mw-redirect" title="Portal:Toolforge/Admin/BotLicensing">/BotLicensing</a></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Sub_pages" data-mw-thread-id="h-Sub_pages"><span data-mw-comment-start="" id="h-Sub_pages"></span>Sub pages<span data-mw-comment-end="h-Sub_pages"></span></h2></div> <div class="mw-prefixindex-body"><ul class="mw-prefixindex-list"><li><a href="/wiki/Portal:Toolforge/Admin/APIs" title="Portal:Toolforge/Admin/APIs">APIs</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Apt_repository" title="Portal:Toolforge/Admin/Apt repository">Apt repository</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Build_Service" title="Portal:Toolforge/Admin/Build Service">Build Service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Envvars_Service" title="Portal:Toolforge/Admin/Envvars Service">Envvars Service</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Exim" title="Portal:Toolforge/Admin/Exim">Exim</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Harbor" title="Portal:Toolforge/Admin/Harbor">Harbor</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Harbor/maintain-harbor" title="Portal:Toolforge/Admin/Harbor/maintain-harbor">Harbor/maintain-harbor</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Infrastructure_tools" title="Portal:Toolforge/Admin/Infrastructure tools">Infrastructure tools</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Jobs_framework" title="Portal:Toolforge/Admin/Jobs framework">Jobs framework</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes" title="Portal:Toolforge/Admin/Kubernetes">Kubernetes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/2020_Kubernetes_cluster_rebuild_plan_notes" title="Portal:Toolforge/Admin/Kubernetes/2020 Kubernetes cluster rebuild plan notes">Kubernetes/2020 Kubernetes cluster rebuild plan notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Certificates" title="Portal:Toolforge/Admin/Kubernetes/Certificates">Kubernetes/Certificates</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Docker-registry" title="Portal:Toolforge/Admin/Kubernetes/Docker-registry">Kubernetes/Docker-registry</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Etcd_(deprecated)" title="Portal:Toolforge/Admin/Kubernetes/Etcd (deprecated)">Kubernetes/Etcd (deprecated)</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Labels" title="Portal:Toolforge/Admin/Kubernetes/Labels">Kubernetes/Labels</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Networking_and_ingress" title="Portal:Toolforge/Admin/Kubernetes/Networking and ingress">Kubernetes/Networking and ingress</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/New_cluster" title="Portal:Toolforge/Admin/Kubernetes/New cluster">Kubernetes/New cluster</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Pod_tracing" title="Portal:Toolforge/Admin/Kubernetes/Pod tracing">Kubernetes/Pod tracing</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/RBAC_and_Pod_security" title="Portal:Toolforge/Admin/Kubernetes/RBAC and Pod security">Kubernetes/RBAC and Pod security</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/RBAC_and_Pod_security/PSP_migration" title="Portal:Toolforge/Admin/Kubernetes/RBAC and Pod security/PSP migration">Kubernetes/RBAC and Pod security/PSP migration</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes">Kubernetes/Upgrading Kubernetes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.21_to_1.22_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.21 to 1.22 notes">Kubernetes/Upgrading Kubernetes/1.21 to 1.22 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.22_to_1.23_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.22 to 1.23 notes">Kubernetes/Upgrading Kubernetes/1.22 to 1.23 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.24_to_1.25_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.24 to 1.25 notes">Kubernetes/Upgrading Kubernetes/1.24 to 1.25 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.25_to_1.26_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.25 to 1.26 notes">Kubernetes/Upgrading Kubernetes/1.25 to 1.26 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.26_to_1.27_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.26 to 1.27 notes">Kubernetes/Upgrading Kubernetes/1.26 to 1.27 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/Upgrading_Kubernetes/1.27_to_1.28_notes" title="Portal:Toolforge/Admin/Kubernetes/Upgrading Kubernetes/1.27 to 1.28 notes">Kubernetes/Upgrading Kubernetes/1.27 to 1.28 notes</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/foxtrot-ldap" title="Portal:Toolforge/Admin/Kubernetes/foxtrot-ldap">Kubernetes/foxtrot-ldap</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Kubernetes/lima-kilo" title="Portal:Toolforge/Admin/Kubernetes/lima-kilo">Kubernetes/lima-kilo</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Legacy_redirector_for_webservices" title="Portal:Toolforge/Admin/Legacy redirector for webservices">Legacy redirector for webservices</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Maintenance" title="Portal:Toolforge/Admin/Maintenance">Maintenance</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting" title="Portal:Toolforge/Admin/Monthly meeting">Monthly meeting</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2022-11-15" title="Portal:Toolforge/Admin/Monthly meeting/2022-11-15">Monthly meeting/2022-11-15</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2022-12-13" title="Portal:Toolforge/Admin/Monthly meeting/2022-12-13">Monthly meeting/2022-12-13</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-01-31" title="Portal:Toolforge/Admin/Monthly meeting/2023-01-31">Monthly meeting/2023-01-31</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-02-21" title="Portal:Toolforge/Admin/Monthly meeting/2023-02-21">Monthly meeting/2023-02-21</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-04-04" title="Portal:Toolforge/Admin/Monthly meeting/2023-04-04">Monthly meeting/2023-04-04</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-05-02" title="Portal:Toolforge/Admin/Monthly meeting/2023-05-02">Monthly meeting/2023-05-02</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-06-06" title="Portal:Toolforge/Admin/Monthly meeting/2023-06-06">Monthly meeting/2023-06-06</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-07-11" title="Portal:Toolforge/Admin/Monthly meeting/2023-07-11">Monthly meeting/2023-07-11</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-09-05" title="Portal:Toolforge/Admin/Monthly meeting/2023-09-05">Monthly meeting/2023-09-05</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-10-03" title="Portal:Toolforge/Admin/Monthly meeting/2023-10-03">Monthly meeting/2023-10-03</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-11-07" title="Portal:Toolforge/Admin/Monthly meeting/2023-11-07">Monthly meeting/2023-11-07</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2023-12-19" title="Portal:Toolforge/Admin/Monthly meeting/2023-12-19">Monthly meeting/2023-12-19</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-01-16" title="Portal:Toolforge/Admin/Monthly meeting/2024-01-16">Monthly meeting/2024-01-16</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-02-13" title="Portal:Toolforge/Admin/Monthly meeting/2024-02-13">Monthly meeting/2024-02-13</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-03-12" title="Portal:Toolforge/Admin/Monthly meeting/2024-03-12">Monthly meeting/2024-03-12</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-04-09" title="Portal:Toolforge/Admin/Monthly meeting/2024-04-09">Monthly meeting/2024-04-09</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-05-14" title="Portal:Toolforge/Admin/Monthly meeting/2024-05-14">Monthly meeting/2024-05-14</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-06-25" title="Portal:Toolforge/Admin/Monthly meeting/2024-06-25">Monthly meeting/2024-06-25</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-07-09" title="Portal:Toolforge/Admin/Monthly meeting/2024-07-09">Monthly meeting/2024-07-09</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-09-03" title="Portal:Toolforge/Admin/Monthly meeting/2024-09-03">Monthly meeting/2024-09-03</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-10-01" title="Portal:Toolforge/Admin/Monthly meeting/2024-10-01">Monthly meeting/2024-10-01</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2024-11-05" title="Portal:Toolforge/Admin/Monthly meeting/2024-11-05">Monthly meeting/2024-11-05</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Monthly_meeting/2025-01-14" title="Portal:Toolforge/Admin/Monthly meeting/2025-01-14">Monthly meeting/2025-01-14</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Packaging" title="Portal:Toolforge/Admin/Packaging">Packaging</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Prometheus" title="Portal:Toolforge/Admin/Prometheus">Prometheus</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Pywikibot_image" title="Portal:Toolforge/Admin/Pywikibot image">Pywikibot image</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Redis" title="Portal:Toolforge/Admin/Redis">Redis</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks" title="Portal:Toolforge/Admin/Runbooks">Runbooks</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/BuildsApiDown" title="Portal:Toolforge/Admin/Runbooks/BuildsApiDown">Runbooks/BuildsApiDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/BuildsApiUpMetricUnknown" title="Portal:Toolforge/Admin/Runbooks/BuildsApiUpMetricUnknown">Runbooks/BuildsApiUpMetricUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/EnvvarsAdmissionDown" title="Portal:Toolforge/Admin/Runbooks/EnvvarsAdmissionDown">Runbooks/EnvvarsAdmissionDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/EnvvarsApiDown" title="Portal:Toolforge/Admin/Runbooks/EnvvarsApiDown">Runbooks/EnvvarsApiDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/EnvvarsApiUpMetricUnknown" title="Portal:Toolforge/Admin/Runbooks/EnvvarsApiUpMetricUnknown">Runbooks/EnvvarsApiUpMetricUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/HarborComponentDown" title="Portal:Toolforge/Admin/Runbooks/HarborComponentDown">Runbooks/HarborComponentDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/HarborDown" title="Portal:Toolforge/Admin/Runbooks/HarborDown">Runbooks/HarborDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/HarborProbeUnknown" title="Portal:Toolforge/Admin/Runbooks/HarborProbeUnknown">Runbooks/HarborProbeUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/JobsApiDown" title="Portal:Toolforge/Admin/Runbooks/JobsApiDown">Runbooks/JobsApiDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/JobsApiUpMetricUnknown" title="Portal:Toolforge/Admin/Runbooks/JobsApiUpMetricUnknown">Runbooks/JobsApiUpMetricUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Kyverno" title="Portal:Toolforge/Admin/Runbooks/Kyverno">Runbooks/Kyverno</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/MaintainKubeusersDown" title="Portal:Toolforge/Admin/Runbooks/MaintainKubeusersDown">Runbooks/MaintainKubeusersDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/PrometheusK8sCertExpirySoon" title="Portal:Toolforge/Admin/Runbooks/PrometheusK8sCertExpirySoon">Runbooks/PrometheusK8sCertExpirySoon</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Redis" title="Portal:Toolforge/Admin/Runbooks/Redis">Runbooks/Redis</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/TektonDown" title="Portal:Toolforge/Admin/Runbooks/TektonDown">Runbooks/TektonDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/TektonUpMetricUnknown" title="Portal:Toolforge/Admin/Runbooks/TektonUpMetricUnknown">Runbooks/TektonUpMetricUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesCapacity" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesCapacity">Runbooks/ToolforgeKubernetesCapacity</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyServerDown" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyServerDown">Runbooks/ToolforgeKubernetesHAproxyServerDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyUnknown" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesHAproxyUnknown">Runbooks/ToolforgeKubernetesHAproxyUnknown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesNodeNotReady" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesNodeNotReady">Runbooks/ToolforgeKubernetesNodeNotReady</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerDiskFull" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerDiskFull">Runbooks/ToolforgeKubernetesWorkerDiskFull</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses" title="Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses">Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Toolforge_Kyverno_low_policy_resources" title="Portal:Toolforge/Admin/Runbooks/Toolforge Kyverno low policy resources">Runbooks/Toolforge Kyverno low policy resources</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Toolforge_Kyverno_no_policy_resources" title="Portal:Toolforge/Admin/Runbooks/Toolforge Kyverno no policy resources">Runbooks/Toolforge Kyverno no policy resources</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/Toolforge_Kyverno_unknown_state" title="Portal:Toolforge/Admin/Runbooks/Toolforge Kyverno unknown state">Runbooks/Toolforge Kyverno unknown state</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsDBReplication" title="Portal:Toolforge/Admin/Runbooks/ToolsDBReplication">Runbooks/ToolsDBReplication</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsNFSDown" title="Portal:Toolforge/Admin/Runbooks/ToolsNFSDown">Runbooks/ToolsNFSDown</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsNfsAlmostFull" title="Portal:Toolforge/Admin/Runbooks/ToolsNfsAlmostFull">Runbooks/ToolsNfsAlmostFull</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/ToolsToolsDBWritableState" title="Portal:Toolforge/Admin/Runbooks/ToolsToolsDBWritableState">Runbooks/ToolsToolsDBWritableState</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Runbooks/k8s-haproxy" title="Portal:Toolforge/Admin/Runbooks/k8s-haproxy">Runbooks/k8s-haproxy</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/SSL_certificates" title="Portal:Toolforge/Admin/SSL certificates">SSL certificates</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker" title="Portal:Toolforge/Admin/Striker">Striker</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker/Build" title="Portal:Toolforge/Admin/Striker/Build">Striker/Build</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker/Deployments" title="Portal:Toolforge/Admin/Striker/Deployments">Striker/Deployments</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Striker/FAQ" title="Portal:Toolforge/Admin/Striker/FAQ">Striker/FAQ</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/System_Overview" title="Portal:Toolforge/Admin/System Overview">System Overview</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolforge-sync-meeting" title="Portal:Toolforge/Admin/Toolforge-sync-meeting">Toolforge-sync-meeting</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolforge_roots_and_Toolforge_admins" title="Portal:Toolforge/Admin/Toolforge roots and Toolforge admins">Toolforge roots and Toolforge admins</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/ToolsDB" title="Portal:Toolforge/Admin/ToolsDB">ToolsDB</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolsbeta" title="Portal:Toolforge/Admin/Toolsbeta">Toolsbeta</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/Toolschecker" title="Portal:Toolforge/Admin/Toolschecker">Toolschecker</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides" title="Portal:Toolforge/Admin/emergency guides">emergency guides</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides/irc_bot_deployment" title="Portal:Toolforge/Admin/emergency guides/irc bot deployment">emergency guides/irc bot deployment</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides/single_tool_webservice" title="Portal:Toolforge/Admin/emergency guides/single tool webservice">emergency guides/single tool webservice</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/emergency_guides/toolforge_down_notification" title="Portal:Toolforge/Admin/emergency guides/toolforge down notification">emergency guides/toolforge down notification</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/local_packages" title="Portal:Toolforge/Admin/local packages">local packages</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/puppet_refactor" title="Portal:Toolforge/Admin/puppet refactor">puppet refactor</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/replagstats" title="Portal:Toolforge/Admin/replagstats">replagstats</a></li> <li><a href="/wiki/Portal:Toolforge/Admin/toolhistory" title="Portal:Toolforge/Admin/toolhistory">toolhistory</a></li> </ul></div>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?useformat=desktop&type=1x1&usesul3=0" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://wikitech.wikimedia.org/w/index.php?title=Portal:Toolforge/Admin&oldid=2256172">https://wikitech.wikimedia.org/w/index.php?title=Portal:Toolforge/Admin&oldid=2256172</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Special:Categories" title="Special:Categories">Categories</a>: <ul><li><a href="/wiki/Category:Toolforge" title="Category:Toolforge">Toolforge</a></li><li><a href="/wiki/Category:Toolforge_admin" title="Category:Toolforge admin">Toolforge admin</a></li><li><a href="/wiki/Category:Cloud_Services_admin" title="Category:Cloud Services admin">Cloud Services admin</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 22 December 2024, at 20:01.</li> <li id="footer-info-copyright">Text is available under the <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike License</a>; additional terms may apply. See <a class="external text" href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use">Terms of Use</a> for details.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Main_Page">About Wikitech</a></li> <li id="footer-places-disclaimers"><a href="https://foundation.wikimedia.org/wiki/General_disclaimer">Disclaimers</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/wikitech.wikimedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//wikitech.wikimedia.org/w/index.php?title=Portal:Toolforge/Admin&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" lang="en" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-header-container vector-sticky-header-container"> <div id="vector-sticky-header" class="vector-sticky-header"> <div class="vector-sticky-header-start"> <div class="vector-sticky-header-icon-start vector-button-flush-left vector-button-flush-right" aria-hidden="true"> <button class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-sticky-header-search-toggle" tabindex="-1" data-event-name="ui.vector-sticky-search-form.icon"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </button> </div> <div role="search" class="vector-search-box-vue vector-search-box-show-thumbnail vector-search-box"> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail"> <form action="/w/index.php" id="vector-sticky-search-form" class="cdx-search-input cdx-search-input--has-end-button"> <div class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikitech"> <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <div class="vector-sticky-header-context-bar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-sticky-header-toc" class="vector-dropdown mw-portlet mw-portlet-sticky-header-toc vector-sticky-header-toc vector-button-flush-left" > <input type="checkbox" id="vector-sticky-header-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-sticky-header-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-sticky-header-toc-label" for="vector-sticky-header-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-sticky-header-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div class="vector-sticky-header-context-bar-primary" aria-hidden="true" ><span class="mw-page-title-main">Portal:Toolforge/Admin</span></div> </div> </div> <div class="vector-sticky-header-end" aria-hidden="true"> <div class="vector-sticky-header-icons"> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-talk-sticky-header" tabindex="-1" data-event-name="talk-sticky-header"><span class="vector-icon mw-ui-icon-speechBubbles mw-ui-icon-wikimedia-speechBubbles"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-subject-sticky-header" tabindex="-1" data-event-name="subject-sticky-header"><span class="vector-icon mw-ui-icon-article mw-ui-icon-wikimedia-article"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-history-sticky-header" tabindex="-1" data-event-name="history-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-history mw-ui-icon-wikimedia-wikimedia-history"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only mw-watchlink" id="ca-watchstar-sticky-header" tabindex="-1" data-event-name="watch-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-star mw-ui-icon-wikimedia-wikimedia-star"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-edit-sticky-header" tabindex="-1" data-event-name="wikitext-edit-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-wikiText mw-ui-icon-wikimedia-wikimedia-wikiText"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-ve-edit-sticky-header" tabindex="-1" data-event-name="ve-edit-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-edit mw-ui-icon-wikimedia-wikimedia-edit"></span> <span></span> </a> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" id="ca-viewsource-sticky-header" tabindex="-1" data-event-name="ve-edit-protected-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-editLock mw-ui-icon-wikimedia-wikimedia-editLock"></span> <span></span> </a> </div> <div class="vector-sticky-header-buttons"> <button class="cdx-button cdx-button--weight-quiet mw-interlanguage-selector" id="p-lang-btn-sticky-header" tabindex="-1" data-event-name="ui.dropdown-p-lang-btn-sticky-header"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>Add languages</span> </button> <a href="#" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--action-progressive" id="ca-addsection-sticky-header" tabindex="-1" data-event-name="addsection-sticky-header"><span class="vector-icon mw-ui-icon-speechBubbleAdd-progressive mw-ui-icon-wikimedia-speechBubbleAdd-progressive"></span> <span>Add topic</span> </a> </div> <div class="vector-sticky-header-icon-end"> <div class="vector-user-links"> </div> </div> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-567b97b4dc-vs6dk","wgBackendResponseTime":388,"wgDiscussionToolsPageThreads":[{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Admin_permissions","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Failover","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-WebProxy-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Static_webserver-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Checker_service-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Redis-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Prometheus-Failover","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Services-Failover","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Command_orchestration","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Administrative_tasks","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Logging_in_as_root-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Disabling_all_ssh_logins_except_root-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Complaints_of_bastion_being_slow-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Local_package_management-Administrative_tasks","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Local_package_policy-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Building_packages-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Deploy_new_misctools_package-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Testing/QA_for_a_new_tools-webservice_package-Local_package_management","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Deploy_new_tools-webservice_package-Local_package_management","replies":[]}]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Webserver_statistics-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Banning_an_IP_from_tool_labs-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Deploying_the_main_web_page-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Regenerate_replica.my.cnf-Administrative_tasks","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Debugging_bad_MariaDB_credentials-Regenerate_replica.my.cnf","replies":[]}]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Regenerate_kubernetes_credentials_for_tools_(.kube/config)-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Adding_K8S_Components-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Deleting_a_tool-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-SSL_certificates-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Granting_a_tool_write_access_to_Elasticsearch-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Package_upgrades-Administrative_tasks","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Creating_a_new_Docker_image_(e.g._for_new_versions_of_Node.js)-Administrative_tasks","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Kubernetes","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Build_service","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Tools-mail_/_Exim","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Users_and_community","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Project_membership_request_approval-Users_and_community","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Quota_management-Users_and_community","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Other","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-How_do_Toolforge_web_services_actually_work?-Other","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-What_makes_a_root/Giving_root_access-Other","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Servicegroup_log-Other","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Useful_administrative_tools-Other","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Brainstorming","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Sub_pages","replies":[]}],"wgPageParseReport":{"discussiontools":{"limitreport-timeusage":"0.030"},"limitreport":{"cputime":"0.149","walltime":"0.284","ppvisitednodes":{"value":2708,"limit":1000000},"postexpandincludesize":{"value":14266,"limit":2097152},"templateargumentsize":{"value":5350,"limit":2097152},"expansiondepth":{"value":13,"limit":100},"expensivefunctioncount":{"value":13,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":38973,"limit":5000000},"timingprofile":["100.00% 203.393 1 -total"," 15.03% 30.561 3 Template:Tracked"," 11.69% 23.781 9 Template:Dir"," 8.00% 16.279 1 Template:Toolforge_nav_admin"," 7.23% 14.714 1 Template:Navigation_sidebar"," 6.68% 13.582 9 Template:Pagelang"," 4.99% 10.150 1 Special:Prefixindex/Portal:Toolforge/Admin/"," 4.89% 9.948 4 Template:Note"," 2.72% 5.529 1 Template:See_also"," 1.69% 3.429 1 Template:See"]},"scribunto":{"limitreport-timeusage":{"value":"0.003","limit":"10.000"},"limitreport-memusage":{"value":622798,"limit":52428800}},"cachereport":{"origin":"mw-web.codfw.main-567b97b4dc-vs6dk","timestamp":"20250212014440","ttl":785,"transientcontent":true}}});});</script> </body> </html>