CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Server Admin Log - Wikitech</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )labswikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat": "dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"424a2ddc-1df3-4536-9fbe-560bc23c5dea","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Server_Admin_Log","wgTitle":"Server Admin Log","wgCurRevisionId":2247060,"wgRevisionId":2247060,"wgArticleId":7919,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["SAL","Operations"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Server_Admin_Log","wgRelevantArticleId":7919,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":["sysop"],"wgNoticeProject":"wikitech","wgCiteReferencePreviewsActive":true,"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgVisualEditor":{"pageLanguageCode":"en", "pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":700000,"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgDiscussionToolsFeaturesEnabled":{"replytool":true,"newtopictool":true,"sourcemodetoolbar":true,"topicsubscription":false,"autotopicsub":false,"visualenhancements":false,"visualenhancements_reply":false,"visualenhancements_pageframe":false},"wgDiscussionToolsFallbackEditMode":"visual","wgULSPosition":"personal","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"wgSiteNoticeId":"2.0"};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready", "user":"ready","user.options":"loading","ext.discussionTools.init.styles":"ready","oojs-ui-core.styles":"ready","oojs-ui.styles.indicators":"ready","mediawiki.widgets.styles":"ready","oojs-ui-core.icons":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.pt":"ready","ext.dismissableSiteNotice.styles":"ready"};RLPAGEMODULES=["site","mediawiki.page.ready","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.site","ext.urlShortener.toolbar","ext.centralauth.centralautologin","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.discussionTools.init","ext.eventLogging","ext.wikimediaEvents","ext.uls.interface","ext.checkUser.clientHints","ext.dismissableSiteNotice"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.discussionTools.init.styles%7Cext.dismissableSiteNotice.styles%7Cext.uls.pt%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cmediawiki.widgets.styles%7Coojs-ui-core.icons%2Cstyles%7Coojs-ui.styles.indicators%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Server Admin Log - Wikitech"> <meta property="og:type" content="website"> <link rel="me" href="https://botsin.space/@wikimedia_sal"> <link rel="icon" href="/static/favicon/wikitech.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikitech (en)"> <link rel="EditURI" type="application/rsd+xml" href="//wikitech.wikimedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://wikitech.wikimedia.org/wiki/Server_Admin_Log"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/"> <link rel="alternate" type="application/atom+xml" title="Wikitech Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="ext-discussiontools-replytool-enabled ext-discussiontools-newtopictool-enabled ext-discussiontools-sourcemodetoolbar-enabled skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Server_Admin_Log rootpage-Server_Admin_Log skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-Server-admin-log:-Prod" class="mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Server admin log: Prod</span></a></li><li id="n-Admin-log:-RelEng" class="mw-list-item"><a href="/wiki/Release_Engineering/SAL"><span>Admin log: RelEng</span></a></li><li id="n-Incident-status" class="mw-list-item"><a href="/wiki/Incident_status"><span>Incident status</span></a></li><li id="n-Deployments" class="mw-list-item"><a href="/wiki/Deployments"><span>Deployments</span></a></li><li id="n-SRE-Team-Help" class="mw-list-item"><a href="/wiki/SRE/SRE_Team_requests"><span>SRE Team Help</span></a></li> </ul> </div> </div> <div id="p-Cloud_VPS_&_Toolforge" class="vector-menu mw-portlet mw-portlet-Cloud_VPS_Toolforge" > <div class="vector-menu-heading"> Cloud VPS & Toolforge </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-Cloud-VPS-portal" class="mw-list-item"><a href="/wiki/Portal:Cloud_VPS"><span>Cloud VPS portal</span></a></li><li id="n-Toolforge-portal" class="mw-list-item"><a href="/wiki/Portal:Toolforge"><span>Toolforge portal</span></a></li><li id="n-Request-VPS-project" class="mw-list-item"><a href="https://phabricator.wikimedia.org/project/view/2875/"><span>Request VPS project</span></a></li><li id="n-Admin-log:-Cloud-VPS" class="mw-list-item"><a href="/wiki/Cloud_VPS_Server_Admin_Log"><span>Admin log: Cloud VPS</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikitech.svg" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikitech" src="/static/images/mobile/copyright/wikitech-wordmark.svg" style="width: 8.75em; height: 1.6875em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikitech [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikitech" aria-label="Search Wikitech" autocapitalize="sentences" title="Search Wikitech [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-uls" class="mw-list-item active user-links-collapsible-item"><a data-mw="interface" href="#" class="uls-trigger cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>English</span></a> </li> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?utm_source=donate&utm_medium=sidebar&utm_campaign=spontaneous&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=Server+Admin+Log" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out user-links-collapsible-item" title="More options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?utm_source=donate&utm_medium=sidebar&utm_campaign=spontaneous&uselang=en"><span>Donate</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=Server+Admin+Log" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"><div id="mw-dismissablenotice-anonplace"></div><script>(function(){var node=document.getElementById("mw-dismissablenotice-anonplace");if(node){node.outerHTML="\u003Cdiv class=\"mw-dismissable-notice\"\u003E\u003Cdiv class=\"mw-dismissable-notice-close\"\u003E[\u003Ca tabindex=\"0\" role=\"button\"\u003Edismiss\u003C/a\u003E]\u003C/div\u003E\u003Cdiv class=\"mw-dismissable-notice-body\"\u003E\u003C!-- CentralNotice --\u003E\u003Cdiv id=\"localNotice\" data-nosnippet=\"\"\u003E\u003Cdiv class=\"sitenotice\" lang=\"en\" dir=\"ltr\"\u003E\u003Ctable style=\"width: 75%; background-color: var(--background-color-warning-subtle, #fdf2d5); border: var(--border-subtle, 1px solid #987027); color: var(--color-base, #202122); border-radius: 10px; padding: 5px; margin: 0 auto;\"\u003E\n\u003Ctbody\u003E\u003Ctr\u003E\n\u003Ctd style=\"width:40px; height:40px; text-align:center; vertical-align:middle; padding: 2px;\"\u003E\u003Cspan typeof=\"mw:File\"\u003E\u003Ca href=\"/wiki/File:OOjs_UI_icon_alert-warning.svg\" class=\"mw-file-description\"\u003E\u003Cimg src=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/30px-OOjs_UI_icon_alert-warning.svg.png\" decoding=\"async\" width=\"30\" height=\"30\" class=\"mw-file-element\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/45px-OOjs_UI_icon_alert-warning.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/60px-OOjs_UI_icon_alert-warning.svg.png 2x\" data-file-width=\"20\" data-file-height=\"20\" /\u003E\u003C/a\u003E\u003C/span\u003E\n\u003C/td\u003E\n\u003Ctd style=\"text-align:center; vertical-align: middle; padding: 4px; max-height: 60px;\"\u003E\u003Cb\u003EWe are migrating Wikitech to \u003Ca href=\"/wiki/Wikitech/SUL-migration\" title=\"Wikitech/SUL-migration\"\u003ESUL\u003C/a\u003E!\u003C/b\u003E\n\u003Cp\u003E\u003Cb\u003EAction may be required for your \u003Ca href=\"/wiki/Wikitech/SUL-migration#What_You_Should_Do\" title=\"Wikitech/SUL-migration\"\u003E account\u003C/a\u003E!\u003C/b\u003E\n\u003C/p\u003E\u003Cp\u003E\u003Cb\u003ETrouble logging in? Please visit \u003Ca href=\"https://phabricator.wikimedia.org/T376267\" class=\"extiw\" title=\"phab:T376267\"\u003ET376267\u003C/a\u003E\u003C/b\u003E\n\u003C/p\u003E\n\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E";}}());</script></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">Beginning</div> </a> </li> <li id="toc-2024-11-23" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-23"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>2024-11-23</span> </div> </a> <ul id="toc-2024-11-23-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-22" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-22"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>2024-11-22</span> </div> </a> <ul id="toc-2024-11-22-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-21" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-21"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>2024-11-21</span> </div> </a> <ul id="toc-2024-11-21-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-20" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-20"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>2024-11-20</span> </div> </a> <ul id="toc-2024-11-20-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-19" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-19"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>2024-11-19</span> </div> </a> <ul id="toc-2024-11-19-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-18" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-18"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>2024-11-18</span> </div> </a> <ul id="toc-2024-11-18-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-17" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-17"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>2024-11-17</span> </div> </a> <ul id="toc-2024-11-17-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-16" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-16"> <div class="vector-toc-text"> <span class="vector-toc-numb">8</span> <span>2024-11-16</span> </div> </a> <ul id="toc-2024-11-16-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-15" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-15"> <div class="vector-toc-text"> <span class="vector-toc-numb">9</span> <span>2024-11-15</span> </div> </a> <ul id="toc-2024-11-15-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-14" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-14"> <div class="vector-toc-text"> <span class="vector-toc-numb">10</span> <span>2024-11-14</span> </div> </a> <ul id="toc-2024-11-14-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-13" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-13"> <div class="vector-toc-text"> <span class="vector-toc-numb">11</span> <span>2024-11-13</span> </div> </a> <ul id="toc-2024-11-13-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-12" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-12"> <div class="vector-toc-text"> <span class="vector-toc-numb">12</span> <span>2024-11-12</span> </div> </a> <ul id="toc-2024-11-12-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-11" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-11"> <div class="vector-toc-text"> <span class="vector-toc-numb">13</span> <span>2024-11-11</span> </div> </a> <ul id="toc-2024-11-11-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-10" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-10"> <div class="vector-toc-text"> <span class="vector-toc-numb">14</span> <span>2024-11-10</span> </div> </a> <ul id="toc-2024-11-10-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-09" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-09"> <div class="vector-toc-text"> <span class="vector-toc-numb">15</span> <span>2024-11-09</span> </div> </a> <ul id="toc-2024-11-09-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-08" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-08"> <div class="vector-toc-text"> <span class="vector-toc-numb">16</span> <span>2024-11-08</span> </div> </a> <ul id="toc-2024-11-08-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-07" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-07"> <div class="vector-toc-text"> <span class="vector-toc-numb">17</span> <span>2024-11-07</span> </div> </a> <ul id="toc-2024-11-07-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-06" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-06"> <div class="vector-toc-text"> <span class="vector-toc-numb">18</span> <span>2024-11-06</span> </div> </a> <ul id="toc-2024-11-06-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-05" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-05"> <div class="vector-toc-text"> <span class="vector-toc-numb">19</span> <span>2024-11-05</span> </div> </a> <ul id="toc-2024-11-05-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-04" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-04"> <div class="vector-toc-text"> <span class="vector-toc-numb">20</span> <span>2024-11-04</span> </div> </a> <ul id="toc-2024-11-04-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-02" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-02"> <div class="vector-toc-text"> <span class="vector-toc-numb">21</span> <span>2024-11-02</span> </div> </a> <ul id="toc-2024-11-02-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-2024-11-01" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#2024-11-01"> <div class="vector-toc-text"> <span class="vector-toc-numb">22</span> <span>2024-11-01</span> </div> </a> <ul id="toc-2024-11-01-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Archives" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Archives"> <div class="vector-toc-text"> <span class="vector-toc-numb">23</span> <span>Archives</span> </div> </a> <ul id="toc-Archives-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Server Admin Log</span></h1> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Server_Admin_Log" title="View the content page [c]" accesskey="c"><span>Page</span></a></li><li id="ca-talk" class="vector-tab-noicon mw-list-item"><a href="/wiki/Talk:Server_Admin_Log" rel="discussion" title="Discussion about the content page [t]" accesskey="t"><span>Discussion</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Read</span></a></li><li id="ca-viewsource" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&action=edit" title="This page is protected.
You can view its source [e]" accesskey="e"><span>View source</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Read</span></a></li><li id="ca-more-viewsource" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&action=edit"><span>View source</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Server_Admin_Log" title="A list of all wiki pages that link here [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Server_Admin_Log" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&oldid=2247060" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=Server_Admin_Log&id=2247060&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FServer_Admin_Log"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FServer_Admin_Log"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-create_a_book" class="mw-list-item"><a href="/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Server+Admin+Log"><span>Create a book</span></a></li><li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=Server_Admin_Log&action=show-download-screen"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Server_Admin_Log&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikitech</div> </div> <div id="contentSub"><div id="mw-content-subtitle"></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><meta property="mw:PageProp/toc"/> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-23" data-mw-thread-id="h-2024-11-23"><span data-mw-comment-start="" id="h-2024-11-23"></span>2024-11-23<span data-mw-comment-end="h-2024-11-23"></span></h2></div> <ul><li>12:08 btullis@cumin1002: END (FAIL) - Cookbook sre.hadoop.roll-restart-masters (exit_code=99) restart masters for Hadoop test cluster: Restart of jvm daemons.</li> <li>12:05 btullis@cumin1002: START - Cookbook sre.hadoop.roll-restart-masters restart masters for Hadoop test cluster: Restart of jvm daemons.</li> <li>02:15 urandom: decommissioning Cassandra/restbase2023-{a,b,c} — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-22" data-mw-thread-id="h-2024-11-22"><span data-mw-comment-start="" id="h-2024-11-22"></span>2024-11-22<span data-mw-comment-end="h-2024-11-22"></span></h2></div> <ul><li>21:51 bking@cumin2002: conftool action : set/pooled=false; selector: dnsdisc=wdqs-internal-scholarly,name=eqiad</li> <li>21:37 bking@cumin2002: conftool action : set/pooled=yes; selector: name=wdqs2026.codfw.wmnet</li> <li>21:37 bking@cumin2002: conftool action : set/pooled=yes; selector: name=wdqs2018.codfw.wmnet</li> <li>21:33 bking@cumin2002: conftool action : set/weight=1; selector: name=wdqs2026.codfw.wmnet</li> <li>21:33 bking@cumin2002: conftool action : set/weight=1; selector: name=wdqs2018.codfw.wmnet</li> <li>21:25 bking@cumin2002: conftool action : set/pooled=yes:weight=1; selector: cluster=wdqs-scholarly,service=wdqs-internal-scholarly</li> <li>21:25 bking@cumin2002: conftool action : set/pooled=yes:weight=1; selector: cluster=wdqs-main,service=wdqs-internal-main</li> <li>20:59 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-worker2005.codfw.wmnet</li> <li>20:59 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-worker2005.codfw.wmnet with OS bookworm</li> <li>20:41 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-worker2005.codfw.wmnet with reason: host reimage</li> <li>20:37 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-worker2005.codfw.wmnet with reason: host reimage</li> <li>20:20 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker2005.codfw.wmnet with OS bookworm</li> <li>20:17 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2005.codfw.wmnet - herron@cumin1002"</li> <li>20:17 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2005.codfw.wmnet - herron@cumin1002"</li> <li>20:17 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-worker2005.codfw.wmnet on all recursors</li> <li>20:17 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-worker2005.codfw.wmnet on all recursors</li> <li>20:17 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:17 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2005.codfw.wmnet - herron@cumin1002"</li> <li>20:17 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2005.codfw.wmnet - herron@cumin1002"</li> <li>20:07 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>20:07 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-worker2005.codfw.wmnet</li> <li>19:47 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-worker2004.codfw.wmnet</li> <li>19:47 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-worker2004.codfw.wmnet with OS bookworm</li> <li>19:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host es2045.codfw.wmnet with OS bookworm</li> <li>19:36 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:36 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host es2046.codfw.wmnet with OS bookworm</li> <li>19:35 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:32 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:32 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host es2043.codfw.wmnet with OS bookworm</li> <li>19:32 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:31 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-worker2004.codfw.wmnet with reason: host reimage</li> <li>19:29 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:27 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-worker2004.codfw.wmnet with reason: host reimage</li> <li>19:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host es2044.codfw.wmnet with OS bookworm</li> <li>19:27 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:26 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:19 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on es2045.codfw.wmnet with reason: host reimage</li> <li>19:16 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on es2046.codfw.wmnet with reason: host reimage</li> <li>19:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on es2043.codfw.wmnet with reason: host reimage</li> <li>19:13 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker2004.codfw.wmnet with OS bookworm</li> <li>19:10 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2004.codfw.wmnet - herron@cumin1002"</li> <li>19:10 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2004.codfw.wmnet - herron@cumin1002"</li> <li>19:10 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-worker2004.codfw.wmnet on all recursors</li> <li>19:10 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-worker2004.codfw.wmnet on all recursors</li> <li>19:10 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>19:10 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2004.codfw.wmnet - herron@cumin1002"</li> <li>19:10 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2004.codfw.wmnet - herron@cumin1002"</li> <li>19:09 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on es2044.codfw.wmnet with reason: host reimage</li> <li>19:05 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on es2045.codfw.wmnet with reason: host reimage</li> <li>19:05 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on es2046.codfw.wmnet with reason: host reimage</li> <li>19:05 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on es2043.codfw.wmnet with reason: host reimage</li> <li>19:05 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on es2044.codfw.wmnet with reason: host reimage</li> <li>18:58 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:58 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-worker2004.codfw.wmnet</li> <li>18:53 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host es2042.codfw.wmnet with OS bookworm</li> <li>18:53 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:52 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:50 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2043.codfw.wmnet with OS bookworm</li> <li>18:50 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2044.codfw.wmnet with OS bookworm</li> <li>18:50 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2045.codfw.wmnet with OS bookworm</li> <li>18:50 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2046.codfw.wmnet with OS bookworm</li> <li>18:45 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-worker2003.codfw.wmnet</li> <li>18:45 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-worker2003.codfw.wmnet with OS bookworm</li> <li>18:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on es2042.codfw.wmnet with reason: host reimage</li> <li>18:32 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on es2042.codfw.wmnet with reason: host reimage</li> <li>18:31 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-worker2003.codfw.wmnet with reason: host reimage</li> <li>18:27 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-worker2003.codfw.wmnet with reason: host reimage</li> <li>18:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2042.codfw.wmnet with OS bookworm</li> <li>18:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2042.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>18:11 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker2003.codfw.wmnet with OS bookworm</li> <li>18:10 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2003.codfw.wmnet - herron@cumin1002"</li> <li>18:10 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2003.codfw.wmnet - herron@cumin1002"</li> <li>18:10 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-worker2003.codfw.wmnet on all recursors</li> <li>18:10 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-worker2003.codfw.wmnet on all recursors</li> <li>18:10 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>18:10 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2003.codfw.wmnet - herron@cumin1002"</li> <li>18:10 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2003.codfw.wmnet - herron@cumin1002"</li> <li>18:09 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2042.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>18:03 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:02 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>18:02 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding es2042 to codfw - jhancock@cumin2002"</li> <li>18:02 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding es2042 to codfw - jhancock@cumin2002"</li> <li>18:02 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-worker2003.codfw.wmnet</li> <li>17:58 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>17:41 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-worker2002.codfw.wmnet</li> <li>17:41 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-worker2002.codfw.wmnet with OS bookworm</li> <li>17:32 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2045.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:31 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2046.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:28 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host es2042</li> <li>17:28 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host es2042</li> <li>17:25 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-worker2002.codfw.wmnet with reason: host reimage</li> <li>17:23 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2044.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:23 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on cloudsw1-d5-eqiad.mgmt,cloudsw1-e4-eqiad.mgmt with reason: replace optics on faulty WMCS link from D5 to E4</li> <li>17:22 cmooney@cumin1002: START - Cookbook sre.hosts.downtime for 1:00:00 on cloudsw1-d5-eqiad.mgmt,cloudsw1-e4-eqiad.mgmt with reason: replace optics on faulty WMCS link from D5 to E4</li> <li>17:22 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-worker2002.codfw.wmnet with reason: host reimage</li> <li>17:20 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2046.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:20 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2045.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:11 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2044.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:09 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2043.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:08 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker2002.codfw.wmnet with OS bookworm</li> <li>17:06 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2002.codfw.wmnet - herron@cumin1002"</li> <li>17:06 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker2002.codfw.wmnet - herron@cumin1002"</li> <li>17:05 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-worker2002.codfw.wmnet on all recursors</li> <li>17:05 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-worker2002.codfw.wmnet on all recursors</li> <li>17:05 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:05 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2002.codfw.wmnet - herron@cumin1002"</li> <li>17:05 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker2002.codfw.wmnet - herron@cumin1002"</li> <li>17:00 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:00 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-worker2002.codfw.wmnet</li> <li>16:57 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2043.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:54 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of aux-k8s-etcd2003.codfw.wmnet to plain</li> <li>16:53 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2042.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:53 herron@cumin1002: START - Cookbook sre.ganeti.changedisk for changing disk type of aux-k8s-etcd2003.codfw.wmnet to plain</li> <li>16:48 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of aux-k8s-etcd2004.codfw.wmnet to plain</li> <li>16:47 herron@cumin1002: START - Cookbook sre.ganeti.changedisk for changing disk type of aux-k8s-etcd2004.codfw.wmnet to plain</li> <li>16:43 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of aux-k8s-etcd2005.codfw.wmnet to plain</li> <li>16:43 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host es2041.codfw.wmnet with OS bookworm</li> <li>16:43 elukey@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>16:43 elukey@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>16:42 herron@cumin1002: START - Cookbook sre.ganeti.changedisk for changing disk type of aux-k8s-etcd2005.codfw.wmnet to plain</li> <li>16:40 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2042.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:27 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on es2041.codfw.wmnet with reason: host reimage</li> <li>16:24 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on es2041.codfw.wmnet with reason: host reimage</li> <li>16:12 claime: homer 'cr*codfw*' commit '<a href="https://phabricator.wikimedia.org/T380473" class="extiw" title="phab:T380473">T380473</a>'</li> <li>16:11 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.decommission (exit_code=1) for hosts parse[2002-2020].codfw.wmnet</li> <li>16:11 cgoubert@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:10 cgoubert@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: parse[2002-2020].codfw.wmnet decommissioned, removing all IPs except the asset tag one - cgoubert@cumin1002"</li> <li>16:10 cgoubert@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: parse[2002-2020].codfw.wmnet decommissioned, removing all IPs except the asset tag one - cgoubert@cumin1002"</li> <li>16:09 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>16:08 bking@deploy2002: Finished deploy [wdqs/wdqs@9927a5a]: 0.3.150 (duration: 03m 00s)</li> <li>16:07 cgoubert@cumin1002: START - Cookbook sre.dns.netbox</li> <li>16:05 bking@deploy2002: Started deploy [wdqs/wdqs@9927a5a]: 0.3.150</li> <li>16:00 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2041.codfw.wmnet with OS bookworm</li> <li>15:31 cgoubert@cumin1002: START - Cookbook sre.hosts.decommission for hosts parse[2002-2020].codfw.wmnet</li> <li>15:31 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>15:29 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts parse2001.codfw.wmnet</li> <li>15:29 cgoubert@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>15:29 cgoubert@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: parse2001.codfw.wmnet decommissioned, removing all IPs except the asset tag one - cgoubert@cumin1002"</li> <li>15:29 cgoubert@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: parse2001.codfw.wmnet decommissioned, removing all IPs except the asset tag one - cgoubert@cumin1002"</li> <li>15:29 elukey@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host es2041.codfw.wmnet with OS bookworm</li> <li>15:25 cgoubert@cumin1002: START - Cookbook sre.dns.netbox</li> <li>15:22 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>15:20 cgoubert@cumin1002: START - Cookbook sre.hosts.decommission for hosts parse2001.codfw.wmnet</li> <li>15:17 ihurbain@deploy2002: helmfile [eqiad] DONE helmfile.d/services/push-notifications: apply</li> <li>15:17 ihurbain@deploy2002: helmfile [eqiad] START helmfile.d/services/push-notifications: apply</li> <li>15:16 cgoubert@deploy2002: helmfile [codfw] DONE helmfile.d/services/push-notifications: apply</li> <li>15:15 cgoubert@deploy2002: helmfile [codfw] START helmfile.d/services/push-notifications: apply</li> <li>15:14 claime: kubectl delete node parse20{01..20}.codfw.wmnet - <a href="https://phabricator.wikimedia.org/T380473" class="extiw" title="phab:T380473">T380473</a></li> <li>15:12 claime: parse[2001-2020].codfw.wmnet 'systemctl stop kubelet.service' - <a href="https://phabricator.wikimedia.org/T380473" class="extiw" title="phab:T380473">T380473</a></li> <li>15:11 claime: parse[2001-2020].codfw.wmnet 'disable-puppet "decom"' - <a href="https://phabricator.wikimedia.org/T380473" class="extiw" title="phab:T380473">T380473</a></li> <li>15:09 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host parse[2001-2020].codfw.wmnet</li> <li>15:02 bking@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on wdqs[2018-2020].codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T379023" class="extiw" title="phab:T379023">T379023</a></li> <li>15:02 bking@cumin2002: START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on wdqs[2018-2020].codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T379023" class="extiw" title="phab:T379023">T379023</a></li> <li>15:01 bking@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on wdqs[2026-2027].codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T379023" class="extiw" title="phab:T379023">T379023</a></li> <li>15:01 bking@cumin2002: START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on wdqs[2026-2027].codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T379023" class="extiw" title="phab:T379023">T379023</a></li> <li>14:54 urandom: decommissioning Cassandra/restbase2022-{a,b,c} —</li> <li>14:53 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2022.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>14:53 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2022.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>14:49 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node depool for host parse[2001-2020].codfw.wmnet</li> <li>14:37 ihurbain@deploy2002: helmfile [codfw] DONE helmfile.d/services/push-notifications: apply</li> <li>14:27 ihurbain@deploy2002: helmfile [codfw] START helmfile.d/services/push-notifications: apply</li> <li>14:23 ihurbain@deploy2002: helmfile [codfw] DONE helmfile.d/services/push-notifications: apply</li> <li>14:22 vgutierrez: restoring haproxykafka on A:cp-ulsfo and A:cp-eqsin - <a href="https://phabricator.wikimedia.org/T380570" class="extiw" title="phab:T380570">T380570</a></li> <li>14:13 ihurbain@deploy2002: helmfile [codfw] START helmfile.d/services/push-notifications: apply</li> <li>14:12 ihurbain@deploy2002: helmfile [staging] DONE helmfile.d/services/push-notifications: apply</li> <li>14:12 ihurbain@deploy2002: helmfile [staging] START helmfile.d/services/push-notifications: apply</li> <li>11:26 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker[2156-2170].codfw.wmnet</li> <li>11:26 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker[2156-2170].codfw.wmnet</li> <li>11:25 claime: homer 'lsw1-d7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:24 claime: homer 'lsw1-d6-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:24 claime: homer 'lsw1-d5-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:23 claime: homer 'lsw1-d4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:22 claime: homer 'lsw1-d1-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:21 claime: homer 'lsw1-c7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:20 claime: homer 'lsw1-c4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:19 claime: homer 'lsw1-c2-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:19 claime: homer 'lsw1-b7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:18 claime: homer 'lsw1-b4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T376966" class="extiw" title="phab:T376966">T376966</a>'</li> <li>11:07 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker2140.codfw.wmnet</li> <li>11:07 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker2140.codfw.wmnet</li> <li>11:04 claime: homer 'lsw1-b7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>11:02 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2159.codfw.wmnet with OS bookworm</li> <li>10:43 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2159.codfw.wmnet with reason: host reimage</li> <li>10:40 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2159.codfw.wmnet with reason: host reimage</li> <li>10:37 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti1014.eqiad.wmnet</li> <li>10:37 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>10:37 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1014.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>10:37 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1014.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>10:31 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>10:26 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti1014.eqiad.wmnet</li> <li>10:23 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti1011.eqiad.wmnet</li> <li>10:23 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>10:23 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1011.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>10:22 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1011.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>10:22 vgutierrez: manually stopping haproxykafka on A:cp-ulsfo and A:cp-eqsin - <a href="https://phabricator.wikimedia.org/T380570" class="extiw" title="phab:T380570">T380570</a></li> <li>10:21 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2159.codfw.wmnet with OS bookworm</li> <li>10:16 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>10:10 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti1011.eqiad.wmnet</li> <li>08:08 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Add sorting options to tree view - oblivian@cumin1002"</li> <li>08:08 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Add sorting options to tree view - oblivian@cumin1002</li> <li>08:07 oblivian@cumin1002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Add sorting options to tree view - oblivian@cumin1002</li> <li>08:07 oblivian@cumin1002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Add sorting options to tree view - oblivian@cumin1002"</li> <li>01:00 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-etcd2005.codfw.wmnet</li> <li>01:00 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-etcd2005.codfw.wmnet with OS bookworm</li> <li>00:46 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-etcd2005.codfw.wmnet with reason: host reimage</li> <li>00:42 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-etcd2005.codfw.wmnet with reason: host reimage</li> <li>00:27 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-etcd2005.codfw.wmnet with OS bookworm</li> <li>00:20 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-etcd2005.codfw.wmnet - herron@cumin1002"</li> <li>00:20 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-etcd2005.codfw.wmnet - herron@cumin1002"</li> <li>00:20 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-etcd2005.codfw.wmnet on all recursors</li> <li>00:20 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-etcd2005.codfw.wmnet on all recursors</li> <li>00:20 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>00:20 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-etcd2005.codfw.wmnet - herron@cumin1002"</li> <li>00:16 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-etcd2005.codfw.wmnet - herron@cumin1002"</li> <li>00:11 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>00:11 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-etcd2005.codfw.wmnet</li> <li>00:11 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-etcd2004.codfw.wmnet</li> <li>00:11 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-etcd2004.codfw.wmnet with OS bookworm</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-21" data-mw-thread-id="h-2024-11-21"><span data-mw-comment-start="" id="h-2024-11-21"></span>2024-11-21<span data-mw-comment-end="h-2024-11-21"></span></h2></div> <ul><li>23:56 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-etcd2004.codfw.wmnet with reason: host reimage</li> <li>23:52 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-etcd2004.codfw.wmnet with reason: host reimage</li> <li>23:36 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-etcd2004.codfw.wmnet with OS bookworm</li> <li>23:29 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-etcd2004.codfw.wmnet - herron@cumin1002"</li> <li>23:29 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-etcd2004.codfw.wmnet - herron@cumin1002"</li> <li>23:29 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-etcd2004.codfw.wmnet on all recursors</li> <li>23:28 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-etcd2004.codfw.wmnet on all recursors</li> <li>23:28 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>23:28 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-etcd2004.codfw.wmnet - herron@cumin1002"</li> <li>23:24 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-etcd2004.codfw.wmnet - herron@cumin1002"</li> <li>23:11 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>23:11 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-etcd2004.codfw.wmnet</li> <li>23:09 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-etcd2003.codfw.wmnet</li> <li>23:09 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-etcd2003.codfw.wmnet with OS bookworm</li> <li>23:08 brennen: end of utc late backport & config window</li> <li>23:07 brennen@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1094005" class="extiw" title="gerrit:1094005">Add statsv to charts impressions (T379833)</a> (duration: 12m 08s)</li> <li>23:06 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2041.codfw.wmnet with OS bookworm</li> <li>23:01 brennen@deploy2002: bvibber, brennen: Continuing with sync</li> <li>23:00 brennen@deploy2002: bvibber, brennen: Backport for <a href="https://gerrit.wikimedia.org/r/1094005" class="extiw" title="gerrit:1094005">Add statsv to charts impressions (T379833)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:55 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1094005" class="extiw" title="gerrit:1094005">Add statsv to charts impressions (T379833)</a></li> <li>22:55 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-etcd2003.codfw.wmnet with reason: host reimage</li> <li>22:54 brennen@deploy2002: Finished scap sync-world: resuming sync for <a href="https://gerrit.wikimedia.org/r/1094000" class="extiw" title="gerrit:1094000">Add tracking categories for {{#chart:}} usage (T369684)</a> after messing up a keypress (duration: 12m 35s)</li> <li>22:52 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-etcd2003.codfw.wmnet with reason: host reimage</li> <li>22:42 brennen@deploy2002: Started scap sync-world: resuming sync for <a href="https://gerrit.wikimedia.org/r/1094000" class="extiw" title="gerrit:1094000">Add tracking categories for {{#chart:}} usage (T369684)</a> after messing up a keypress</li> <li>22:40 brennen@deploy2002: Sync cancelled.</li> <li>22:40 brennen@deploy2002: bvibber, brennen: Backport for <a href="https://gerrit.wikimedia.org/r/1094000" class="extiw" title="gerrit:1094000">Add tracking categories for {{#chart:}} usage (T369684)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:38 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-etcd2003.codfw.wmnet with OS bookworm</li> <li>22:36 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-etcd2003.codfw.wmnet - herron@cumin1002"</li> <li>22:36 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-etcd2003.codfw.wmnet - herron@cumin1002"</li> <li>22:35 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-etcd2003.codfw.wmnet on all recursors</li> <li>22:35 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-etcd2003.codfw.wmnet on all recursors</li> <li>22:35 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>22:35 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-etcd2003.codfw.wmnet - herron@cumin1002"</li> <li>22:35 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-etcd2003.codfw.wmnet - herron@cumin1002"</li> <li>22:32 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>22:32 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-etcd2003.codfw.wmnet</li> <li>22:25 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1094000" class="extiw" title="gerrit:1094000">Add tracking categories for {{#chart:}} usage (T369684)</a></li> <li>22:25 brennen@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092334" class="extiw" title="gerrit:1092334">Disable various extensions when using the shared login domain (T373737)</a> (duration: 18m 16s)</li> <li>22:22 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>22:18 brennen@deploy2002: tgr, brennen: Continuing with sync</li> <li>22:10 brennen@deploy2002: tgr, brennen: Backport for <a href="https://gerrit.wikimedia.org/r/1092334" class="extiw" title="gerrit:1092334">Disable various extensions when using the shared login domain (T373737)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:06 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092334" class="extiw" title="gerrit:1092334">Disable various extensions when using the shared login domain (T373737)</a></li> <li>22:05 brennen@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1094047" class="extiw" title="gerrit:1094047">Revert "Reduce number of bucketsizes for MediaViewer (group0)" (T372165)</a> (duration: 10m 34s)</li> <li>21:58 brennen@deploy2002: brennen: Continuing with sync</li> <li>21:58 brennen@deploy2002: brennen: Backport for <a href="https://gerrit.wikimedia.org/r/1094047" class="extiw" title="gerrit:1094047">Revert "Reduce number of bucketsizes for MediaViewer (group0)" (T372165)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:54 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1094047" class="extiw" title="gerrit:1094047">Revert "Reduce number of bucketsizes for MediaViewer (group0)" (T372165)</a></li> <li>21:51 brennen@deploy2002: Sync cancelled.</li> <li>21:42 brennen@deploy2002: brennen, tgr, simon04: Backport for <a href="https://gerrit.wikimedia.org/r/1079640" class="extiw" title="gerrit:1079640">Reduce number of bucketsizes for MediaViewer (group0) (T372165)</a>, <a href="https://gerrit.wikimedia.org/r/1093961" class="extiw" title="gerrit:1093961">Set 'remember' central session object field when recreating (T379254 T372702)</a>, <a href="https://gerrit.wikimedia.org/r/1093962" class="extiw" title="gerrit:1093962">Use cookie to access central session when local session expired</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:39 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1079640" class="extiw" title="gerrit:1079640">Reduce number of bucketsizes for MediaViewer (group0) (T372165)</a>, <a href="https://gerrit.wikimedia.org/r/1093961" class="extiw" title="gerrit:1093961">Set 'remember' central session object field when recreating (T379254 T372702)</a>, <a href="https://gerrit.wikimedia.org/r/1093962" class="extiw" title="gerrit:1093962">Use cookie to access central session when local session expired</a></li> <li>21:36 brennen@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093960" class="extiw" title="gerrit:1093960">Enable Skin-Codex logging (T375287)</a> (duration: 15m 53s)</li> <li>21:29 brennen@deploy2002: brennen, jdlrobson: Continuing with sync</li> <li>21:26 brennen@deploy2002: brennen, jdlrobson: Backport for <a href="https://gerrit.wikimedia.org/r/1093960" class="extiw" title="gerrit:1093960">Enable Skin-Codex logging (T375287)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:20 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093960" class="extiw" title="gerrit:1093960">Enable Skin-Codex logging (T375287)</a></li> <li>21:19 brennen@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090968" class="extiw" title="gerrit:1090968">Enable AutoModerator on afwiki (T376597)</a> (duration: 13m 50s)</li> <li>21:12 brennen@deploy2002: kgraessle, brennen: Continuing with sync</li> <li>21:10 brennen@deploy2002: kgraessle, brennen: Backport for <a href="https://gerrit.wikimedia.org/r/1090968" class="extiw" title="gerrit:1090968">Enable AutoModerator on afwiki (T376597)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:05 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090968" class="extiw" title="gerrit:1090968">Enable AutoModerator on afwiki (T376597)</a></li> <li>20:46 tgr</li> <li>20:24 sukhe@puppetserver1001: conftool action : set/pooled=yes; selector: name=cp2038.codfw.wmnet [reason: DIMM replaced, <a href="https://phabricator.wikimedia.org/T308459" class="extiw" title="phab:T308459">T308459</a>]</li> <li>20:20 sukhe: force agent on cp2038</li> <li>19:31 gmodena@deploy2002: Finished deploy [analytics/refinery@199401a] (hadoop-test): Ad-hoc deployment TEST [analytics/refinery@199401a6] (duration: 03m 45s)</li> <li>19:27 gmodena@deploy2002: Started deploy [analytics/refinery@199401a] (hadoop-test): Ad-hoc deployment TEST [analytics/refinery@199401a6]</li> <li>19:07 gmodena@deploy2002: Finished deploy [analytics/refinery@199401a] (thin): Ad-hoc deployment THIN [analytics/refinery@199401a6] (duration: 05m 37s)</li> <li>19:01 gmodena@deploy2002: Started deploy [analytics/refinery@199401a] (thin): Ad-hoc deployment THIN [analytics/refinery@199401a6]</li> <li>18:57 gmodena@deploy2002: Finished deploy [analytics/refinery@199401a]: Ad-hoc deployment [analytics/refinery@199401a6] (duration: 14m 08s)</li> <li>18:57 cdanis@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093983" class="extiw" title="gerrit:1093983">Follow-up fix for Charts enable on commons/test2 (T379689)</a> (duration: 11m 29s)</li> <li>18:49 cdanis@deploy2002: cdanis, bvibber: Continuing with sync</li> <li>18:49 cdanis@deploy2002: cdanis, bvibber: Backport for <a href="https://gerrit.wikimedia.org/r/1093983" class="extiw" title="gerrit:1093983">Follow-up fix for Charts enable on commons/test2 (T379689)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>18:45 cdanis@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093983" class="extiw" title="gerrit:1093983">Follow-up fix for Charts enable on commons/test2 (T379689)</a></li> <li>18:43 gmodena@deploy2002: Started deploy [analytics/refinery@199401a]: Ad-hoc deployment [analytics/refinery@199401a6]</li> <li>18:21 cdanis@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091328" class="extiw" title="gerrit:1091328">Enabling Charts on commons+test2 (T379689)</a> (duration: 14m 05s)</li> <li>18:16 jayme@cumin2002: conftool action : set/pooled=yes; selector: name=kubestage200[34].codfw.wmnet</li> <li>18:15 jayme@cumin2002: conftool action : set/weight=10; selector: name=kubestage200[34].codfw.wmnet</li> <li>18:13 cdanis@deploy2002: cdanis, bvibber: Continuing with sync</li> <li>18:12 cdanis@deploy2002: cdanis, bvibber: Backport for <a href="https://gerrit.wikimedia.org/r/1091328" class="extiw" title="gerrit:1091328">Enabling Charts on commons+test2 (T379689)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>18:10 sukhe: running puppet on A:cp to resolve failed puppet run</li> <li>18:10 sukhe: sudo cumin -b11 'A:cp' 'run-puppet-agent</li> <li>18:09 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on cp2038.codfw.wmnet with reason: DIMM replacement in progress</li> <li>18:09 sukhe@cumin1002: START - Cookbook sre.hosts.downtime for 1:00:00 on cp2038.codfw.wmnet with reason: DIMM replacement in progress</li> <li>18:07 cdanis@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091328" class="extiw" title="gerrit:1091328">Enabling Charts on commons+test2 (T379689)</a></li> <li>17:58 sukhe@puppetserver1001: conftool action : set/pooled=no; selector: name=cp2038.codfw.wmnet [reason: DIMM failure <a href="https://phabricator.wikimedia.org/T308459" class="extiw" title="phab:T308459">T308459</a>]</li> <li>17:45 jayme@cumin2002: END (FAIL) - Cookbook sre.k8s.pool-depool-node (exit_code=99) check for host kubestage2003.codfw.wmnet</li> <li>17:45 jayme@cumin2002: START - Cookbook sre.k8s.pool-depool-node check for host kubestage2003.codfw.wmnet</li> <li>17:40 andrew@cumin1002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts clouddb2002-dev.codfw.wmnet</li> <li>17:40 andrew@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:40 andrew@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: clouddb2002-dev.codfw.wmnet decommissioned, removing all IPs except the asset tag one - andrew@cumin1002"</li> <li>17:39 andrew@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: clouddb2002-dev.codfw.wmnet decommissioned, removing all IPs except the asset tag one - andrew@cumin1002"</li> <li>17:39 fabfur: adding acls to kafka-jumbo cluster (<a href="https://phabricator.wikimedia.org/T380373" class="extiw" title="phab:T380373">T380373</a>)</li> <li>17:36 andrew@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:31 andrew@cumin1002: START - Cookbook sre.hosts.decommission for hosts clouddb2002-dev.codfw.wmnet</li> <li>17:02 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2157.codfw.wmnet with OS bookworm</li> <li>16:54 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for lvs2013.codfw.wmnet</li> <li>16:54 sukhe@cumin1002: START - Cookbook sre.hosts.remove-downtime for lvs2013.codfw.wmnet</li> <li>16:54 sukhe: enable puppet on lvs2013 and start pybal</li> <li>16:48 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on lvs2013.codfw.wmnet with reason: rebooting</li> <li>16:47 sukhe@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on lvs2013.codfw.wmnet with reason: rebooting</li> <li>16:47 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>16:47 cgoubert@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - cgoubert@cumin1002"</li> <li>16:46 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs2013.codfw.wmnet</li> <li>16:46 cgoubert@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - cgoubert@cumin1002"</li> <li>16:43 sukhe@cumin1002: START - Cookbook sre.hosts.reboot-single for host lvs2013.codfw.wmnet</li> <li>16:43 sukhe: rebooting drained lvs2013</li> <li>16:43 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2157.codfw.wmnet with reason: host reimage</li> <li>16:39 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2157.codfw.wmnet with reason: host reimage</li> <li>16:26 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2140.codfw.wmnet with reason: host reimage</li> <li>16:23 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2140.codfw.wmnet with reason: host reimage</li> <li>16:21 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2157.codfw.wmnet with OS bookworm</li> <li>16:20 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2157.codfw.wmnet with OS bookworm</li> <li>16:13 sukhe@puppetserver1001: conftool action : set/pooled=no; selector: name=cluster=dnsbox,dc=magru [reason: testing]</li> <li>16:08 dancy@deploy2002: Finished scap sync-world: testing (duration: 03m 01s)</li> <li>16:05 dancy@deploy2002: Started scap sync-world: testing</li> <li>16:04 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>16:03 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>16:00 dancy@deploy2002: Installing scap version "4.127.0" for 209 hosts</li> <li>15:39 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093927" class="extiw" title="gerrit:1093927">Fix layout broken by display:flex on HorizontalLayout (T380471)</a>, <a href="https://gerrit.wikimedia.org/r/1093928" class="extiw" title="gerrit:1093928">Revert "ExperimentUserDefaultsManager: use read latest when retrieving central id"</a> (duration: 15m 51s)</li> <li>15:34 gmodena@deploy2002: Finished deploy [analytics/refinery@358ccf5] (hadoop-test): Ad-hoc deployment TEST [analytics/refinery@358ccf55] (duration: 03m 30s)</li> <li>15:33 kartik@deploy2002: abi, sgimeno, kartik: Continuing with sync</li> <li>15:31 gmodena@deploy2002: Started deploy [analytics/refinery@358ccf5] (hadoop-test): Ad-hoc deployment TEST [analytics/refinery@358ccf55]</li> <li>15:29 gmodena@deploy2002: Finished deploy [analytics/refinery@358ccf5] (thin): Ad-hoc deployment THIN [analytics/refinery@358ccf55] (duration: 05m 16s)</li> <li>15:29 ihurbain@deploy2002: helmfile [eqiad] DONE helmfile.d/services/push-notifications: apply</li> <li>15:29 kartik@deploy2002: abi, sgimeno, kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1093927" class="extiw" title="gerrit:1093927">Fix layout broken by display:flex on HorizontalLayout (T380471)</a>, <a href="https://gerrit.wikimedia.org/r/1093928" class="extiw" title="gerrit:1093928">Revert "ExperimentUserDefaultsManager: use read latest when retrieving central id"</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:28 ihurbain@deploy2002: helmfile [eqiad] START helmfile.d/services/push-notifications: apply</li> <li>15:28 ihurbain@deploy2002: helmfile [codfw] DONE helmfile.d/services/push-notifications: apply</li> <li>15:27 ihurbain@deploy2002: helmfile [codfw] START helmfile.d/services/push-notifications: apply</li> <li>15:26 ebernhardson@deploy2002: Finished deploy [airflow-dags/search@6183645]: increase driver memory for mjolnir feature selection (duration: 00m 31s)</li> <li>15:26 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on lvs2013.codfw.wmnet with reason: rebooting</li> <li>15:25 sukhe@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on lvs2013.codfw.wmnet with reason: rebooting</li> <li>15:25 ebernhardson@deploy2002: Started deploy [airflow-dags/search@6183645]: increase driver memory for mjolnir feature selection</li> <li>15:24 sukhe: stop pybal on lvs2013 to confirm changes in CR <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1091243"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1091243/">1091243</a></span></li> <li>15:24 gmodena@deploy2002: Started deploy [analytics/refinery@358ccf5] (thin): Ad-hoc deployment THIN [analytics/refinery@358ccf55]</li> <li>15:24 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093927" class="extiw" title="gerrit:1093927">Fix layout broken by display:flex on HorizontalLayout (T380471)</a>, <a href="https://gerrit.wikimedia.org/r/1093928" class="extiw" title="gerrit:1093928">Revert "ExperimentUserDefaultsManager: use read latest when retrieving central id"</a></li> <li>15:23 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:23 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:16 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:15 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:11 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2021.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>15:10 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2021.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>15:06 gmodena@deploy2002: Finished deploy [analytics/refinery@358ccf5]: Ad-hoc deployment [analytics/refinery@358ccf55] (duration: 11m 44s)</li> <li>14:56 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2169.codfw.wmnet with OS bookworm</li> <li>14:55 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>14:54 gmodena@deploy2002: Started deploy [analytics/refinery@358ccf5]: Ad-hoc deployment [analytics/refinery@358ccf55]</li> <li>14:53 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2168.codfw.wmnet with OS bookworm</li> <li>14:51 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2170.codfw.wmnet with OS bookworm</li> <li>14:50 sergi0: UTC afternoon deploys done</li> <li>14:49 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2167.codfw.wmnet with OS bookworm</li> <li>14:48 sgimeno@deploy2002: Sync cancelled.</li> <li>14:47 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>14:47 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2166.codfw.wmnet with OS bookworm</li> <li>14:43 jynus@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on kafka-main1001.eqiad.wmnet with reason: Per claime's recommendation</li> <li>14:43 jynus@cumin1002: START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on kafka-main1001.eqiad.wmnet with reason: Per claime's recommendation</li> <li>14:43 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2157.codfw.wmnet with OS bookworm</li> <li>14:41 sgimeno@deploy2002: sgimeno: Backport for <a href="https://gerrit.wikimedia.org/r/1093889" class="extiw" title="gerrit:1093889">ExperimentUserDefaultsManager: use read latest when retrieving central id (T379682)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:39 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>14:36 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2169.codfw.wmnet with reason: host reimage</li> <li>14:35 sgimeno@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093889" class="extiw" title="gerrit:1093889">ExperimentUserDefaultsManager: use read latest when retrieving central id (T379682)</a></li> <li>14:33 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2168.codfw.wmnet with reason: host reimage</li> <li>14:31 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2170.codfw.wmnet with reason: host reimage</li> <li>14:28 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2167.codfw.wmnet with reason: host reimage</li> <li>14:25 ihurbain@deploy2002: helmfile [staging] DONE helmfile.d/services/push-notifications: apply</li> <li>14:25 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2166.codfw.wmnet with reason: host reimage</li> <li>14:25 ihurbain@deploy2002: helmfile [staging] START helmfile.d/services/push-notifications: apply</li> <li>14:24 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2170.codfw.wmnet with reason: host reimage</li> <li>14:24 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2169.codfw.wmnet with reason: host reimage</li> <li>14:23 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2168.codfw.wmnet with reason: host reimage</li> <li>14:23 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2167.codfw.wmnet with reason: host reimage</li> <li>14:22 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2166.codfw.wmnet with reason: host reimage</li> <li>14:21 sgimeno@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092956" class="extiw" title="gerrit:1092956">enwiki: Add abusefilter-access-protected-vars to EFH/EFM (T380332)</a> (duration: 13m 50s)</li> <li>14:14 sgimeno@deploy2002: eggroll97, sgimeno: Continuing with sync</li> <li>14:11 sgimeno@deploy2002: eggroll97, sgimeno: Backport for <a href="https://gerrit.wikimedia.org/r/1092956" class="extiw" title="gerrit:1092956">enwiki: Add abusefilter-access-protected-vars to EFH/EFM (T380332)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:11 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kubestage1006.eqiad.wmnet with OS bookworm</li> <li>14:07 sgimeno@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092956" class="extiw" title="gerrit:1092956">enwiki: Add abusefilter-access-protected-vars to EFH/EFM (T380332)</a></li> <li>14:06 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kubestage1005.eqiad.wmnet with OS bookworm</li> <li>14:05 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2170.codfw.wmnet with OS bookworm</li> <li>14:05 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2169.codfw.wmnet with OS bookworm</li> <li>14:04 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2168.codfw.wmnet with OS bookworm</li> <li>14:04 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2167.codfw.wmnet with OS bookworm</li> <li>14:03 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2166.codfw.wmnet with OS bookworm</li> <li>13:54 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubestage1006.eqiad.wmnet with reason: host reimage</li> <li>13:51 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on kubestage1006.eqiad.wmnet with reason: host reimage</li> <li>13:47 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubestage1005.eqiad.wmnet with reason: host reimage</li> <li>13:44 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on kubestage1005.eqiad.wmnet with reason: host reimage</li> <li>13:34 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host kubestage1006.eqiad.wmnet with OS bookworm</li> <li>13:33 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.rename (exit_code=0) from kubernetes1008 to kubestage1006</li> <li>13:32 jayme@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host kubestage1006</li> <li>13:31 jayme@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host kubestage1006</li> <li>13:31 jayme@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>13:31 jayme@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Renaming kubernetes1008 to kubestage1006 - jayme@cumin2002"</li> <li>13:30 jayme@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Renaming kubernetes1008 to kubestage1006 - jayme@cumin2002"</li> <li>13:27 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host kubestage1005.eqiad.wmnet with OS bookworm</li> <li>13:25 jayme@cumin2002: START - Cookbook sre.dns.netbox</li> <li>13:25 jayme@cumin2002: START - Cookbook sre.hosts.rename from kubernetes1008 to kubestage1006</li> <li>13:24 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.rename (exit_code=0) from kubernetes1007 to kubestage1005</li> <li>13:24 jayme@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host kubestage1005</li> <li>13:22 jayme@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host kubestage1005</li> <li>13:22 jayme@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>13:22 jayme@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Renaming kubernetes1007 to kubestage1005 - jayme@cumin2002"</li> <li>13:21 jayme@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Renaming kubernetes1007 to kubestage1005 - jayme@cumin2002"</li> <li>13:18 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2160.codfw.wmnet with OS bookworm</li> <li>13:18 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-ats (exit_code=0) Rolling upgrade/restart of Apache Traffic Server on P{cp5026*} and A:cp for 9.2.6-1wm2</li> <li>13:17 jayme@cumin2002: START - Cookbook sre.dns.netbox</li> <li>13:17 jayme@cumin2002: START - Cookbook sre.hosts.rename from kubernetes1007 to kubestage1005</li> <li>13:14 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2164.codfw.wmnet with OS bookworm</li> <li>13:14 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-ats Rolling upgrade/restart of Apache Traffic Server on P{cp5026*} and A:cp for 9.2.6-1wm2</li> <li>13:14 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-ats (exit_code=0) Rolling upgrade/restart of Apache Traffic Server on P{cp5018*} and A:cp for 9.2.6-1wm2</li> <li>13:11 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2162.codfw.wmnet with OS bookworm</li> <li>13:10 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-ats Rolling upgrade/restart of Apache Traffic Server on P{cp5018*} and A:cp for 9.2.6-1wm2</li> <li>13:10 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2165.codfw.wmnet with OS bookworm</li> <li>13:05 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>13:02 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2158.codfw.wmnet with OS bookworm</li> <li>12:58 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2161.codfw.wmnet with OS bookworm</li> <li>12:58 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2160.codfw.wmnet with reason: host reimage</li> <li>12:55 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2156.codfw.wmnet with OS bookworm</li> <li>12:55 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2164.codfw.wmnet with reason: host reimage</li> <li>12:52 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2162.codfw.wmnet with reason: host reimage</li> <li>12:49 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2165.codfw.wmnet with reason: host reimage</li> <li>12:46 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2163.codfw.wmnet with reason: host reimage</li> <li>12:42 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2158.codfw.wmnet with reason: host reimage</li> <li>12:39 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2161.codfw.wmnet with reason: host reimage</li> <li>12:38 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2165.codfw.wmnet with reason: host reimage</li> <li>12:38 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2164.codfw.wmnet with reason: host reimage</li> <li>12:38 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2163.codfw.wmnet with reason: host reimage</li> <li>12:37 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2162.codfw.wmnet with reason: host reimage</li> <li>12:36 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2156.codfw.wmnet with reason: host reimage</li> <li>12:36 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2160.codfw.wmnet with reason: host reimage</li> <li>12:35 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2161.codfw.wmnet with reason: host reimage</li> <li>12:32 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2158.codfw.wmnet with reason: host reimage</li> <li>12:32 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2156.codfw.wmnet with reason: host reimage</li> <li>12:19 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2165.codfw.wmnet with OS bookworm</li> <li>12:18 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2164.codfw.wmnet with OS bookworm</li> <li>12:18 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>12:17 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2162.codfw.wmnet with OS bookworm</li> <li>12:17 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2160.codfw.wmnet with OS bookworm</li> <li>12:16 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2161.codfw.wmnet with OS bookworm</li> <li>12:16 jmm@deploy2002: helmfile [eqiad] DONE helmfile.d/services/thumbor: apply</li> <li>12:13 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2158.codfw.wmnet with OS bookworm</li> <li>12:13 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2156.codfw.wmnet with OS bookworm</li> <li>12:09 jmm@deploy2002: helmfile [eqiad] START helmfile.d/services/thumbor: apply</li> <li>12:09 jmm@deploy2002: helmfile [codfw] DONE helmfile.d/services/thumbor: apply</li> <li>12:02 jmm@deploy2002: helmfile [codfw] START helmfile.d/services/thumbor: apply</li> <li>11:56 jmm@deploy2002: helmfile [staging] DONE helmfile.d/services/thumbor: apply</li> <li>11:56 jmm@deploy2002: helmfile [staging] START helmfile.d/services/thumbor: apply</li> <li>11:00 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host thanos-be1005.eqiad.wmnet with OS bullseye</li> <li>11:00 elukey@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>10:59 elukey@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>10:41 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host kubernetes[1007-1008].eqiad.wmnet</li> <li>10:41 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on thanos-be1005.eqiad.wmnet with reason: host reimage</li> <li>10:40 jayme@cumin2002: START - Cookbook sre.k8s.pool-depool-node depool for host kubernetes[1007-1008].eqiad.wmnet</li> <li>10:39 urbanecm@deploy2002: helmfile [codfw] DONE helmfile.d/services/linkrecommendation: apply</li> <li>10:38 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71113">https://phabricator.wikimedia.org/P71113</a> and previous config saved to /var/cache/conftool/dbconfig/20241121-103834-arnaudb.json</li> <li>10:38 urbanecm@deploy2002: helmfile [codfw] START helmfile.d/services/linkrecommendation: apply</li> <li>10:38 urbanecm@deploy2002: helmfile [eqiad] DONE helmfile.d/services/linkrecommendation: apply</li> <li>10:37 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on thanos-be1005.eqiad.wmnet with reason: host reimage</li> <li>10:36 urbanecm@deploy2002: helmfile [eqiad] START helmfile.d/services/linkrecommendation: apply</li> <li>10:34 urbanecm@deploy2002: helmfile [staging] DONE helmfile.d/services/linkrecommendation: apply</li> <li>10:33 urbanecm@deploy2002: helmfile [staging] START helmfile.d/services/linkrecommendation: apply</li> <li>10:25 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host thanos-be1005.eqiad.wmnet with OS bullseye</li> <li>10:23 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71112">https://phabricator.wikimedia.org/P71112</a> and previous config saved to /var/cache/conftool/dbconfig/20241121-102328-arnaudb.json</li> <li>10:19 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.debug (exit_code=0) for Netbox circuit ID 102</li> <li>10:19 ayounsi@cumin1002: START - Cookbook sre.network.debug for Netbox circuit ID 102</li> <li>10:08 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71111">https://phabricator.wikimedia.org/P71111</a> and previous config saved to /var/cache/conftool/dbconfig/20241121-100821-arnaudb.json</li> <li>10:01 dcausse@deploy2002: helmfile [codfw] DONE helmfile.d/services/eventgate-main: sync</li> <li>10:01 dcausse@deploy2002: helmfile [codfw] START helmfile.d/services/eventgate-main: sync</li> <li>09:59 dcausse: restarting eventgate-main@codfw</li> <li>09:53 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71110">https://phabricator.wikimedia.org/P71110</a> and previous config saved to /var/cache/conftool/dbconfig/20241121-095313-arnaudb.json</li> <li>09:51 arnaudb@cumin1002: dbctl commit (dc=all): 'Depooling db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71109">https://phabricator.wikimedia.org/P71109</a> and previous config saved to /var/cache/conftool/dbconfig/20241121-095102-arnaudb.json</li> <li>09:50 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2187.codfw.wmnet with reason: Maintenance</li> <li>09:50 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 8:00:00 on db2187.codfw.wmnet with reason: Maintenance</li> <li>09:50 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2155.codfw.wmnet with reason: Maintenance</li> <li>09:50 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on db2155.codfw.wmnet with reason: Maintenance</li> <li>09:35 moritzm: installing nghttp2 security updates</li> <li>09:18 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db1246.eqiad.wmnet with OS bookworm</li> <li>09:17 aklapper@deploy2002: rebuilt and synchronized wikiversions files: group2 to 1.44.0-wmf.4 refs <a href="https://phabricator.wikimedia.org/T375663" class="extiw" title="phab:T375663">T375663</a></li> <li>09:07 moritzm: installing exim4 security updates</li> <li>09:03 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db1246.eqiad.wmnet with reason: host reimage</li> <li>09:00 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on db1246.eqiad.wmnet with reason: host reimage</li> <li>08:45 arnaudb@cumin1002: START - Cookbook sre.hosts.reimage for host db1246.eqiad.wmnet with OS bookworm</li> <li>08:21 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093733" class="extiw" title="gerrit:1093733">Enable the Contribute menu in 4th group of Wikis (T375303)</a> (duration: 14m 05s)</li> <li>08:14 kartik@deploy2002: kartik: Continuing with sync</li> <li>08:10 kartik@deploy2002: kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1093733" class="extiw" title="gerrit:1093733">Enable the Contribute menu in 4th group of Wikis (T375303)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:06 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093733" class="extiw" title="gerrit:1093733">Enable the Contribute menu in 4th group of Wikis (T375303)</a></li> <li>07:48 moritzm: removing ganeti1017 from active Ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>05:51 aikochou@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'article-models' for release 'main' .</li> <li>02:30 brett: Import libvmod-re2_2.0.0-2~bpo11u1 into varnish-staging apt component</li> <li>00:45 urandom: decommissioning Cassandra/restbase2021-{a,b,c} — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:42 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2023.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:42 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2023.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:42 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2022.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:42 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2022.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:42 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2021.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:42 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2021.codfw.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>00:40 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for restbase2038.codfw.wmnet</li> <li>00:40 eevans@cumin1002: START - Cookbook sre.hosts.remove-downtime for restbase2038.codfw.wmnet</li> <li>00:40 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for restbase2037.codfw.wmnet</li> <li>00:40 eevans@cumin1002: START - Cookbook sre.hosts.remove-downtime for restbase2037.codfw.wmnet</li> <li>00:40 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for restbase2036.codfw.wmnet</li> <li>00:40 eevans@cumin1002: START - Cookbook sre.hosts.remove-downtime for restbase2036.codfw.wmnet</li> <li>00:15 urbanecm: [urbanecm@deploy2002 ~]$ mwscript-k8s -- extensions/GrowthExperiments/maintenance/revalidateLinkRecommendations.php --wiki=azwiki --all --verbose # <a href="https://phabricator.wikimedia.org/T380329" class="extiw" title="phab:T380329">T380329</a></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-20" data-mw-thread-id="h-2024-11-20"><span data-mw-comment-start="" id="h-2024-11-20"></span>2024-11-20<span data-mw-comment-end="h-2024-11-20"></span></h2></div> <ul><li>23:22 cjming: end of UTC late backport window</li> <li>23:20 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7c940d6f"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7c940d6f">7c940d6f</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="3311520a"><a class="external text" href="https://gerrit.wikimedia.org/r/q/3311520a">3311520a</a></span></li> <li>23:17 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093408" class="extiw" title="gerrit:1093408">Temporarily disable dark mode for anonymous users (T379765)</a> (duration: 13m 06s)</li> <li>23:10 cjming@deploy2002: jdlrobson, cjming: Continuing with sync</li> <li>23:08 cjming@deploy2002: jdlrobson, cjming: Backport for <a href="https://gerrit.wikimedia.org/r/1093408" class="extiw" title="gerrit:1093408">Temporarily disable dark mode for anonymous users (T379765)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>23:04 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093408" class="extiw" title="gerrit:1093408">Temporarily disable dark mode for anonymous users (T379765)</a></li> <li>23:03 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093328" class="extiw" title="gerrit:1093328">knwiki: update portal namespace (T380366)</a> (duration: 12m 17s)</li> <li>22:56 cjming@deploy2002: cjming, anzx: Continuing with sync</li> <li>22:55 cjming@deploy2002: cjming, anzx: Backport for <a href="https://gerrit.wikimedia.org/r/1093328" class="extiw" title="gerrit:1093328">knwiki: update portal namespace (T380366)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:52 brett: Import libvmod-querysort 0.4-3 into varnish-staging apt component</li> <li>22:51 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093328" class="extiw" title="gerrit:1093328">knwiki: update portal namespace (T380366)</a></li> <li>22:49 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093446" class="extiw" title="gerrit:1093446">Revert "Add contact form for U4C"</a> (duration: 14m 22s)</li> <li>22:49 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>22:41 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:41 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:40 cjming@deploy2002: trainbranchbot, cjming: Continuing with sync</li> <li>22:40 cjming@deploy2002: trainbranchbot, cjming: Backport for <a href="https://gerrit.wikimedia.org/r/1093446" class="extiw" title="gerrit:1093446">Revert "Add contact form for U4C"</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:39 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:39 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:34 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093446" class="extiw" title="gerrit:1093446">Revert "Add contact form for U4C"</a></li> <li>22:31 cjming@deploy2002: Sync cancelled.</li> <li>22:28 cjming@deploy2002: nmw03, cjming: Backport for <a href="https://gerrit.wikimedia.org/r/1091868" class="extiw" title="gerrit:1091868">Add contact form for U4C (T379317)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:27 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on thanos-be2005.codfw.wmnet with reason: host reimage</li> <li>22:24 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on thanos-be2005.codfw.wmnet with reason: host reimage</li> <li>22:23 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:22 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091868" class="extiw" title="gerrit:1091868">Add contact form for U4C (T379317)</a></li> <li>22:21 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:20 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093358" class="extiw" title="gerrit:1093358">Bump wikimedia/parsoid to 0.21.0-a7 (T373776 T380333)</a>, <a href="https://gerrit.wikimedia.org/r/1093359" class="extiw" title="gerrit:1093359">Bump wikimedia/parsoid to 0.21.0-a7 (T380333)</a> (duration: 17m 11s)</li> <li>22:18 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:16 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:13 cjming@deploy2002: arlolra, cjming: Continuing with sync</li> <li>22:12 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>22:11 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>22:11 jhathaway@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhathaway@cumin2002"</li> <li>22:09 jhathaway@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhathaway@cumin2002"</li> <li>22:08 cjming@deploy2002: arlolra, cjming: Backport for <a href="https://gerrit.wikimedia.org/r/1093358" class="extiw" title="gerrit:1093358">Bump wikimedia/parsoid to 0.21.0-a7 (T373776 T380333)</a>, <a href="https://gerrit.wikimedia.org/r/1093359" class="extiw" title="gerrit:1093359">Bump wikimedia/parsoid to 0.21.0-a7 (T380333)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:06 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>22:03 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093358" class="extiw" title="gerrit:1093358">Bump wikimedia/parsoid to 0.21.0-a7 (T373776 T380333)</a>, <a href="https://gerrit.wikimedia.org/r/1093359" class="extiw" title="gerrit:1093359">Bump wikimedia/parsoid to 0.21.0-a7 (T380333)</a></li> <li>22:02 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>21:52 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>21:50 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>21:47 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on thanos-be2005.codfw.wmnet with reason: host reimage</li> <li>21:43 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on thanos-be2005.codfw.wmnet with reason: host reimage</li> <li>21:40 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>21:32 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>21:31 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>21:28 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091810" class="extiw" title="gerrit:1091810">[ptwiki] Enable the CampaignEvents extension (T380090)</a> (duration: 15m 04s)</li> <li>21:23 eileen: * civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="e29243f0"><a class="external text" href="https://gerrit.wikimedia.org/r/q/e29243f0">e29243f0</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7c940d6f"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7c940d6f">7c940d6f</a></span></li> <li>21:20 cjming@deploy2002: cjming, albertoleoncio: Continuing with sync</li> <li>21:19 cjming@deploy2002: cjming, albertoleoncio: Backport for <a href="https://gerrit.wikimedia.org/r/1091810" class="extiw" title="gerrit:1091810">[ptwiki] Enable the CampaignEvents extension (T380090)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:13 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091810" class="extiw" title="gerrit:1091810">[ptwiki] Enable the CampaignEvents extension (T380090)</a></li> <li>21:08 dancy@deploy2002: Installing scap version "4.124.0" for 209 hosts</li> <li>21:06 dancy@deploy2002: Installing scap version "4.124.0" for 209 hosts</li> <li>21:05 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-ctrl2003.codfw.wmnet</li> <li>21:05 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-ctrl2003.codfw.wmnet with OS bookworm</li> <li>21:03 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>21:00 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:51 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-ctrl2003.codfw.wmnet with reason: host reimage</li> <li>20:48 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>20:48 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>20:48 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-ctrl2003.codfw.wmnet with reason: host reimage</li> <li>20:48 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>20:47 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2041.codfw.wmnet with OS bookworm</li> <li>20:44 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>20:40 dancy@deploy2002: Installation of scap version "4.126.0" completed for 1 hosts</li> <li>20:39 dancy@deploy2002: Installing scap version "4.126.0" for 1 hosts</li> <li>20:32 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-ctrl2003.codfw.wmnet with OS bookworm</li> <li>20:30 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:30 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:28 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-ctrl2003.codfw.wmnet - herron@cumin1002"</li> <li>20:28 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-ctrl2003.codfw.wmnet - herron@cumin1002"</li> <li>20:28 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-ctrl2003.codfw.wmnet on all recursors</li> <li>20:28 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-ctrl2003.codfw.wmnet on all recursors</li> <li>20:28 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:28 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-ctrl2003.codfw.wmnet - herron@cumin1002"</li> <li>20:26 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-ctrl2003.codfw.wmnet - herron@cumin1002"</li> <li>20:13 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>20:13 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-ctrl2003.codfw.wmnet</li> <li>20:10 dancy@deploy2002: Installing scap version "4.126.0" for 1 hosts</li> <li>20:08 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:05 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:03 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>19:52 hashar@deploy2002: Finished deploy [integration/docroot@1627206]: build: update mediawiki-codesniffer to 45.0.0 & prevent LibUp from removing a phpcs rule (duration: 00m 10s)</li> <li>19:52 hashar@deploy2002: Started deploy [integration/docroot@1627206]: build: update mediawiki-codesniffer to 45.0.0 & prevent LibUp from removing a phpcs rule</li> <li>19:51 dancy@deploy2002: Installing scap version "4.126.0" for 1 hosts</li> <li>19:47 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>19:42 dancy@deploy2002: Installing scap version "4.126.0" for 209 hosts</li> <li>19:35 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-ctrl2002.codfw.wmnet</li> <li>19:35 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-ctrl2002.codfw.wmnet with OS bookworm</li> <li>19:20 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-ctrl2002.codfw.wmnet with reason: host reimage</li> <li>19:17 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-ctrl2002.codfw.wmnet with reason: host reimage</li> <li>19:12 urandom: bootstrapping cassandra, restbase2038-{a,b,c} — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>19:08 inflatador: bking@krb1001 add kerberos keytab for blunderbuss <a class="external free" href="https://phabricator.wikimedia.org/P71106">https://phabricator.wikimedia.org/P71106</a> <a href="https://phabricator.wikimedia.org/T371994" class="extiw" title="phab:T371994">T371994</a></li> <li>19:04 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-ctrl2002.codfw.wmnet with OS bookworm</li> <li>19:03 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-ctrl2002.codfw.wmnet - herron@cumin1002"</li> <li>19:03 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-ctrl2002.codfw.wmnet - herron@cumin1002"</li> <li>19:03 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-ctrl2002.codfw.wmnet on all recursors</li> <li>19:03 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-ctrl2002.codfw.wmnet on all recursors</li> <li>19:03 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>19:03 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-ctrl2002.codfw.wmnet - herron@cumin1002"</li> <li>19:03 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-ctrl2002.codfw.wmnet - herron@cumin1002"</li> <li>18:58 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:58 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-ctrl2002.codfw.wmnet</li> <li>17:32 joal@deploy2002: Finished deploy [analytics/refinery@295d5a4] (hadoop-test): Regular analytics weekly train BIS TEST [analytics/refinery@295d5a44] (duration: 03m 36s)</li> <li>17:28 joal@deploy2002: Started deploy [analytics/refinery@295d5a4] (hadoop-test): Regular analytics weekly train BIS TEST [analytics/refinery@295d5a44]</li> <li>17:28 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>17:27 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>17:22 joal@deploy2002: Finished deploy [analytics/refinery@295d5a4] (thin): Regular analytics weekly train BIS THIN [analytics/refinery@295d5a44] (duration: 05m 02s)</li> <li>17:22 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>17:21 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>17:20 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>17:19 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>17:18 joal@deploy2002: Started deploy [analytics/refinery@295d5a4] (thin): Regular analytics weekly train BIS THIN [analytics/refinery@295d5a44]</li> <li>17:17 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>17:16 joal@deploy2002: Finished deploy [analytics/refinery@295d5a4]: Regular analytics weekly train BIS [analytics/refinery@295d5a44] (duration: 03m 41s)</li> <li>17:12 joal@deploy2002: Started deploy [analytics/refinery@295d5a4]: Regular analytics weekly train BIS [analytics/refinery@295d5a44]</li> <li>17:05 sukhe: restart tomcat on idp2004</li> <li>17:04 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>17:03 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>17:02 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>17:01 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>17:00 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>17:00 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>16:43 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop: apply</li> <li>16:43 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop: apply</li> <li>16:43 jiji@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop: apply</li> <li>16:43 jiji@deploy2002: helmfile [staging] START helmfile.d/services/changeprop: apply</li> <li>16:43 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/eventstreams: apply</li> <li>16:42 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/tegola-vector-tiles: apply</li> <li>16:40 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/tegola-vector-tiles: apply</li> <li>16:39 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-page-content-change-enrich: apply</li> <li>16:38 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-page-content-change-enrich: apply</li> <li>16:37 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/eventstreams: apply</li> <li>16:36 jiji@deploy2002: helmfile [staging] DONE helmfile.d/services/eventstreams: apply</li> <li>16:35 klausman@deploy2002: helmfile [ml-staging-codfw] DONE helmfile.d/admin 'apply'.</li> <li>16:35 jiji@deploy2002: helmfile [staging] START helmfile.d/services/eventstreams: apply</li> <li>16:34 klausman@deploy2002: helmfile [ml-staging-codfw] START helmfile.d/admin 'apply'.</li> <li>16:28 jiji@deploy2002: helmfile [staging] START helmfile.d/services/eventgate-main: apply</li> <li>16:26 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>16:25 aikochou@deploy2002: helmfile [ml-serve-codfw] Ran 'sync' command on namespace 'revision-models' for release 'main' .</li> <li>16:24 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>16:23 jiji@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>16:22 jiji@deploy2002: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>16:22 jiji@deploy2002: helmfile [staging] DONE helmfile.d/services/benthos-cache-invalidator: apply</li> <li>16:21 jiji@deploy2002: helmfile [staging] START helmfile.d/services/benthos-cache-invalidator: apply</li> <li>16:15 aikochou@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revision-models' for release 'main' .</li> <li>16:10 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1017.eqiad.wmnet</li> <li>15:51 apine@deploy2002: helmfile [eqiad] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:50 apine@deploy2002: helmfile [eqiad] START helmfile.d/services/wikifunctions: apply</li> <li>15:50 apine@deploy2002: helmfile [codfw] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:49 apine@deploy2002: helmfile [codfw] START helmfile.d/services/wikifunctions: apply</li> <li>15:48 dancy@deploy2002: Finished scap sync-world: no-op deployment for testing. (duration: 03m 21s)</li> <li>15:44 dancy@deploy2002: Started scap sync-world: no-op deployment for testing.</li> <li>15:44 apine@deploy2002: helmfile [staging] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:44 apine@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>15:37 apine@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>15:37 apine@deploy2002: helmfile [staging] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:33 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1206.eqiad.wmnet with reason: host overworked by dumps - <a href="https://phabricator.wikimedia.org/T368098" class="extiw" title="phab:T368098">T368098</a></li> <li>15:33 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1206.eqiad.wmnet with reason: host overworked by dumps - <a href="https://phabricator.wikimedia.org/T368098" class="extiw" title="phab:T368098">T368098</a></li> <li>15:31 jynus: starting resharding of commons backup files into new host backup2010 <a href="https://phabricator.wikimedia.org/T376892" class="extiw" title="phab:T376892">T376892</a></li> <li>15:27 apine@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>15:23 apine@deploy2002: helmfile [codfw] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:23 apine@deploy2002: helmfile [codfw] START helmfile.d/services/wikifunctions: apply</li> <li>15:22 apine@deploy2002: helmfile [eqiad] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:22 apine@deploy2002: helmfile [eqiad] START helmfile.d/services/wikifunctions: apply</li> <li>15:19 apine@deploy2002: helmfile [staging] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:19 apine@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>15:15 apine@deploy2002: helmfile [eqiad] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:14 apine@deploy2002: helmfile [eqiad] START helmfile.d/services/wikifunctions: apply</li> <li>15:13 apine@deploy2002: helmfile [codfw] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:13 apine@deploy2002: helmfile [codfw] START helmfile.d/services/wikifunctions: apply</li> <li>15:10 apine@deploy2002: helmfile [staging] DONE helmfile.d/services/wikifunctions: apply</li> <li>15:09 apine@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>15:09 urandom: bootstrapping cassandra, restbase2037-{a,b,c} — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>15:04 btullis@cumin1002: END (PASS) - Cookbook sre.ceph.roll-restart-reboot-server (exit_code=0) rolling reboot on P{cephosd100[2-4].eqiad.wmnet} and (A:cephosd)</li> <li>14:57 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>14:53 JennH: power cycling unresponsive mgmt switch in codfw: msw-c3-codfw</li> <li>14:50 btullis@cumin1002: END (FAIL) - Cookbook sre.hadoop.roll-restart-workers (exit_code=99) restart workers for Hadoop analytics cluster: Roll restart of jvm daemons for openjdk upgrade.</li> <li>14:43 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>14:29 cdanis: <a href="https://phabricator.wikimedia.org/T380226" class="extiw" title="phab:T380226">T380226</a> 💙cdanis@mwmaint2002.codfw.wmnet ~ 🕤☕ mwscript sql.php --wiki=commonswiki --cluster=extension1 /srv/mediawiki/php-1.44.0-wmf.4/extensions/JsonConfig/sql/mysql/tables-generated.sql</li> <li>14:25 sukhe@puppetserver1001: conftool action : set/pooled=yes; selector: name=cp7007.magru.wmnet [reason: host reimaged]</li> <li>14:24 btullis@cumin1002: START - Cookbook sre.ceph.roll-restart-reboot-server rolling reboot on P{cephosd100[2-4].eqiad.wmnet} and (A:cephosd)</li> <li>14:23 jynus: starting resharding of commons backup files into new host backup1010 <a href="https://phabricator.wikimedia.org/T376892" class="extiw" title="phab:T376892">T376892</a></li> <li>14:23 sukhe: running homer on asw*magru*</li> <li>14:06 jiji@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>14:05 jiji@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>14:05 jiji@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>14:05 jiji@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>14:05 jiji@deploy2002: helmfile [ml-staging-codfw] DONE helmfile.d/admin 'apply'.</li> <li>14:04 jiji@deploy2002: helmfile [ml-staging-codfw] START helmfile.d/admin 'apply'.</li> <li>14:04 jiji@deploy2002: helmfile [ml-serve-codfw] DONE helmfile.d/admin 'apply'.</li> <li>14:04 jiji@deploy2002: helmfile [ml-serve-codfw] START helmfile.d/admin 'apply'.</li> <li>14:04 jiji@deploy2002: helmfile [ml-serve-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>14:03 jiji@deploy2002: helmfile [ml-serve-eqiad] START helmfile.d/admin 'apply'.</li> <li>14:03 jiji@deploy2002: helmfile [staging-codfw] DONE helmfile.d/admin 'apply'.</li> <li>14:03 jiji@deploy2002: helmfile [staging-codfw] START helmfile.d/admin 'apply'.</li> <li>14:03 jiji@deploy2002: helmfile [staging-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>14:03 jiji@deploy2002: helmfile [staging-eqiad] START helmfile.d/admin 'apply'.</li> <li>14:03 jiji@deploy2002: helmfile [codfw] DONE helmfile.d/admin 'apply'.</li> <li>14:02 jiji@deploy2002: helmfile [codfw] START helmfile.d/admin 'apply'.</li> <li>14:02 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/admin 'apply'.</li> <li>14:02 jiji@deploy2002: helmfile [eqiad] START helmfile.d/admin 'apply'.</li> <li>13:56 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker[2136-2139,2141-2155].codfw.wmnet</li> <li>13:55 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker[2136-2139,2141-2155].codfw.wmnet</li> <li>13:53 claime: homer 'lsw1-d4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:52 claime: homer 'lsw1-b4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:52 claime: homer 'lsw1-d2-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:51 claime: homer 'lsw1-c2-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:50 claime: homer 'lsw1-d7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:50 claime: homer 'lsw1-c4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:49 claime: homer 'lsw1-d5-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:48 claime: homer 'lsw1-b7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:47 claime: homer 'lsw1-c7-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:46 claime: homer 'lsw1-d6-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:45 claime: homer 'lsw1-b2-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:44 claime: homer 'lsw1-d1-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377028" class="extiw" title="phab:T377028">T377028</a>'</li> <li>13:41 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2151.codfw.wmnet with OS bookworm</li> <li>13:38 effie: putting kafka-main1006.eqiad.wmnet in production</li> <li>13:38 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2152.codfw.wmnet with OS bookworm</li> <li>13:36 jiji@cumin1002: END (PASS) - Cookbook sre.kafka.roll-restart-reboot-brokers (exit_code=0) rolling restart_daemons on A:kafka-main-eqiad</li> <li>13:33 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2154.codfw.wmnet with OS bookworm</li> <li>13:31 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2155.codfw.wmnet with OS bookworm</li> <li>13:29 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>13:28 btullis@cumin1002: START - Cookbook sre.hadoop.roll-restart-workers restart workers for Hadoop analytics cluster: Roll restart of jvm daemons for openjdk upgrade.</li> <li>13:28 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>13:26 jiji@cumin1002: START - Cookbook sre.kafka.roll-restart-reboot-brokers rolling restart_daemons on A:kafka-main-eqiad</li> <li>13:26 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2153.codfw.wmnet with OS bookworm</li> <li>13:23 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2150.codfw.wmnet with OS bookworm</li> <li>13:21 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2151.codfw.wmnet with reason: host reimage</li> <li>13:17 sukhe@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp7007.magru.wmnet with OS bullseye</li> <li>13:17 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2152.codfw.wmnet with reason: host reimage</li> <li>13:14 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2154.codfw.wmnet with reason: host reimage</li> <li>13:11 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2155.codfw.wmnet with reason: host reimage</li> <li>13:07 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2153.codfw.wmnet with reason: host reimage</li> <li>13:03 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2150.codfw.wmnet with reason: host reimage</li> <li>13:02 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2155.codfw.wmnet with reason: host reimage</li> <li>13:02 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2154.codfw.wmnet with reason: host reimage</li> <li>13:01 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1017.eqiad.wmnet</li> <li>13:01 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2153.codfw.wmnet with reason: host reimage</li> <li>13:01 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2152.codfw.wmnet with reason: host reimage</li> <li>13:00 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2151.codfw.wmnet with reason: host reimage</li> <li>13:00 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2150.codfw.wmnet with reason: host reimage</li> <li>12:55 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1017.eqiad.wmnet</li> <li>12:51 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>12:50 sukhe@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cp7007.magru.wmnet with reason: host reimage</li> <li>12:50 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>12:49 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1017.eqiad.wmnet</li> <li>12:46 sukhe@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on cp7007.magru.wmnet with reason: host reimage</li> <li>12:44 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2155.codfw.wmnet with OS bookworm</li> <li>12:43 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2154.codfw.wmnet with OS bookworm</li> <li>12:42 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2153.codfw.wmnet with OS bookworm</li> <li>12:42 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2152.codfw.wmnet with OS bookworm</li> <li>12:41 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2143.codfw.wmnet with OS bookworm</li> <li>12:41 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2151.codfw.wmnet with OS bookworm</li> <li>12:41 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2150.codfw.wmnet with OS bookworm</li> <li>12:39 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2146.codfw.wmnet with OS bookworm</li> <li>12:38 sukhe: re-enable puppet on cumin2002</li> <li>12:34 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>12:34 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2145.codfw.wmnet with OS bookworm</li> <li>12:33 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>12:31 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2147.codfw.wmnet with OS bookworm</li> <li>12:26 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2148.codfw.wmnet with OS bookworm</li> <li>12:23 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2149.codfw.wmnet with OS bookworm</li> <li>12:23 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>12:22 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>12:22 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2143.codfw.wmnet with reason: host reimage</li> <li>12:21 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2144.codfw.wmnet with OS bookworm</li> <li>12:20 sukhe@cumin2002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>12:19 sukhe@cumin2002: END (FAIL) - Cookbook sre.hosts.dhcp (exit_code=99) for host cp7007.magru.wmnet</li> <li>12:18 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2146.codfw.wmnet with reason: host reimage</li> <li>12:16 sukhe@cumin2002: START - Cookbook sre.hosts.dhcp for host cp7007.magru.wmnet</li> <li>12:16 sukhe@cumin1002: END (FAIL) - Cookbook sre.hosts.dhcp (exit_code=99) for host cp7007.magru.wmnet</li> <li>12:15 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2145.codfw.wmnet with reason: host reimage</li> <li>12:14 sukhe@cumin1002: START - Cookbook sre.hosts.dhcp for host cp7007.magru.wmnet</li> <li>12:11 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2147.codfw.wmnet with reason: host reimage</li> <li>12:08 sukhe: disable puppet on cumin2002 to test cumin alias for A:installserver</li> <li>12:07 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2148.codfw.wmnet with reason: host reimage</li> <li>12:04 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2149.codfw.wmnet with reason: host reimage</li> <li>12:01 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2144.codfw.wmnet with reason: host reimage</li> <li>11:59 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2149.codfw.wmnet with reason: host reimage</li> <li>11:59 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2148.codfw.wmnet with reason: host reimage</li> <li>11:58 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2147.codfw.wmnet with reason: host reimage</li> <li>11:57 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2146.codfw.wmnet with reason: host reimage</li> <li>11:57 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2145.codfw.wmnet with reason: host reimage</li> <li>11:56 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2143.codfw.wmnet with reason: host reimage</li> <li>11:56 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2144.codfw.wmnet with reason: host reimage</li> <li>11:40 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2149.codfw.wmnet with OS bookworm</li> <li>11:39 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2148.codfw.wmnet with OS bookworm</li> <li>11:39 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2147.codfw.wmnet with OS bookworm</li> <li>11:38 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2146.codfw.wmnet with OS bookworm</li> <li>11:38 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2145.codfw.wmnet with OS bookworm</li> <li>11:37 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2144.codfw.wmnet with OS bookworm</li> <li>11:36 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2143.codfw.wmnet with OS bookworm</li> <li>11:30 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_magru</li> <li>11:24 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_magru</li> <li>11:22 akosiaris: decommission cxserver endpoints /api/rest_v1/transform/html/from, /api/rest_v1/transform/word/from from RESTBase <a href="https://phabricator.wikimedia.org/T375616" class="extiw" title="phab:T375616">T375616</a></li> <li>10:43 btullis@cumin1002: END (PASS) - Cookbook sre.ceph.roll-restart-reboot-server (exit_code=0) rolling reboot on P{cephosd1001.eqiad.wmnet} and (A:cephosd)</li> <li>10:38 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_magru</li> <li>10:38 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_magru</li> <li>10:37 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_esams</li> <li>10:34 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_esams</li> <li>10:33 btullis@cumin1002: START - Cookbook sre.ceph.roll-restart-reboot-server rolling reboot on P{cephosd1001.eqiad.wmnet} and (A:cephosd)</li> <li>10:33 jiji@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on kafka-main[1001,1006].eqiad.wmnet with reason: Hardware refresh</li> <li>10:33 jayme: re-enabled puppet on all k8s controll planes for rollout of <a href="https://phabricator.wikimedia.org/T380142" class="extiw" title="phab:T380142">T380142</a></li> <li>10:33 jiji@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on kafka-main[1001,1006].eqiad.wmnet with reason: Hardware refresh</li> <li>10:22 effie: removing leadership from kafka-main1001 - <a href="https://phabricator.wikimedia.org/T363214" class="extiw" title="phab:T363214">T363214</a></li> <li>10:19 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>10:18 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:52 aklapper@deploy2002: rebuilt and synchronized wikiversions files: group1 to 1.44.0-wmf.4 refs <a href="https://phabricator.wikimedia.org/T375663" class="extiw" title="phab:T375663">T375663</a></li> <li>09:44 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:44 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:41 kevinbazira@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'experimental' for release 'main' .</li> <li>09:38 akosiaris: decommission cxserver endpoints /api/rest_v1/list/(pair|tool|languagepairs) from RESTBase <a href="https://phabricator.wikimedia.org/T375616" class="extiw" title="phab:T375616">T375616</a></li> <li>09:35 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:34 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:33 aklapper@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093172" class="extiw" title="gerrit:1093172">EditionLookup: Update EntityLookup calls (T380304)</a> (duration: 13m 33s)</li> <li>09:33 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_esams</li> <li>09:33 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_esams</li> <li>09:28 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:27 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:27 aklapper@deploy2002: aklapper, thiemowmde: Continuing with sync</li> <li>09:26 aklapper@deploy2002: aklapper, thiemowmde: Backport for <a href="https://gerrit.wikimedia.org/r/1093172" class="extiw" title="gerrit:1093172">EditionLookup: Update EntityLookup calls (T380304)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>09:21 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of prometheus7001.magru.wmnet to plain</li> <li>09:20 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of prometheus7001.magru.wmnet to plain</li> <li>09:20 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:20 aklapper@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1093172" class="extiw" title="gerrit:1093172">EditionLookup: Update EntityLookup calls (T380304)</a></li> <li>09:19 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>09:18 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of doh7002.wikimedia.org to plain</li> <li>09:15 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of doh7002.wikimedia.org to plain</li> <li>09:13 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ncredir7002.magru.wmnet to plain</li> <li>09:13 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ncredir7002.magru.wmnet to plain</li> <li>08:56 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of durum7002.magru.wmnet to plain</li> <li>08:51 jayme: disabling puppet on all k8s controll planes for rollout of <a href="https://phabricator.wikimedia.org/T380142" class="extiw" title="phab:T380142">T380142</a></li> <li>08:48 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of durum7002.magru.wmnet to plain</li> <li>08:46 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of bast7001.wikimedia.org to plain</li> <li>08:44 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of bast7001.wikimedia.org to plain</li> <li>08:35 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti7004.magru.wmnet</li> <li>08:35 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti7004.magru.wmnet</li> <li>08:35 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti7004.magru.wmnet</li> <li>08:34 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti7004.magru.wmnet</li> <li>08:18 hashar: Restarted CI Jenkins to upgrade Leastload plugin and remove the SSH server plugin</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-19" data-mw-thread-id="h-2024-11-19"><span data-mw-comment-start="" id="h-2024-11-19"></span>2024-11-19<span data-mw-comment-end="h-2024-11-19"></span></h2></div> <ul><li>22:50 ryankemper@deploy2002: Started deploy [wdqs/wdqs@9927a5a] (wcqs): Deploy 0.3.150 to WCQS</li> <li>22:00 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092341" class="extiw" title="gerrit:1092341">Enable experimental Parsoid fragment support on labs and test wikis (T374661)</a>, <a href="https://gerrit.wikimedia.org/r/1092850" class="extiw" title="gerrit:1092850">Revert "editcheck: Remove try/catch around transaction squashing" (T333710 T380234)</a>, <a href="https://gerrit.wikimedia.org/r/1092851" class="extiw" title="gerrit:1092851">Revert "editcheck: Remove try/catch around transaction squashing" (T333710 T380234)</a> (duration: 20m 39s)</li> <li>21:53 urbanecm@deploy2002: cscott, kemayo, urbanecm: Continuing with sync</li> <li>21:45 urbanecm@deploy2002: cscott, kemayo, urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1092341" class="extiw" title="gerrit:1092341">Enable experimental Parsoid fragment support on labs and test wikis (T374661)</a>, <a href="https://gerrit.wikimedia.org/r/1092850" class="extiw" title="gerrit:1092850">Revert "editcheck: Remove try/catch around transaction squashing" (T333710 T380234)</a>, <a href="https://gerrit.wikimedia.org/r/1092851" class="extiw" title="gerrit:1092851">Revert "editcheck: Remove try/catch around transaction squashing" (T333710 T380234)</a> synced to the testservers (<a rel="nofollow" class="external free" href="https://wikitech.wikimedia.or">https://wikitech.wikimedia.or</a></li> <li>21:39 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2041.codfw.wmnet with OS bookworm</li> <li>21:39 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092341" class="extiw" title="gerrit:1092341">Enable experimental Parsoid fragment support on labs and test wikis (T374661)</a>, <a href="https://gerrit.wikimedia.org/r/1092850" class="extiw" title="gerrit:1092850">Revert "editcheck: Remove try/catch around transaction squashing" (T333710 T380234)</a>, <a href="https://gerrit.wikimedia.org/r/1092851" class="extiw" title="gerrit:1092851">Revert "editcheck: Remove try/catch around transaction squashing" (T333710 T380234)</a></li> <li>21:38 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092296" class="extiw" title="gerrit:1092296">Promote Vector 2022 as default on 3 wikis (T379765)</a>, <a href="https://gerrit.wikimedia.org/r/1092912" class="extiw" title="gerrit:1092912">Separate cache key space for test & production JsonConfig data (T380320)</a> (duration: 14m 38s)</li> <li>21:31 urbanecm@deploy2002: bvibber, jdlrobson, urbanecm: Continuing with sync</li> <li>21:29 urbanecm@deploy2002: bvibber, jdlrobson, urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1092296" class="extiw" title="gerrit:1092296">Promote Vector 2022 as default on 3 wikis (T379765)</a>, <a href="https://gerrit.wikimedia.org/r/1092912" class="extiw" title="gerrit:1092912">Separate cache key space for test & production JsonConfig data (T380320)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:23 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092296" class="extiw" title="gerrit:1092296">Promote Vector 2022 as default on 3 wikis (T379765)</a>, <a href="https://gerrit.wikimedia.org/r/1092912" class="extiw" title="gerrit:1092912">Separate cache key space for test & production JsonConfig data (T380320)</a></li> <li>21:16 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2038.codfw.wmnet with reason: Bootstrapping — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>21:15 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2038.codfw.wmnet with reason: Bootstrapping — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>21:15 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2037.codfw.wmnet with reason: Bootstrapping — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>21:15 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2037.codfw.wmnet with reason: Bootstrapping — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>21:15 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase2036.codfw.wmnet with reason: Bootstrapping — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>21:15 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase2036.codfw.wmnet with reason: Bootstrapping — <a href="https://phabricator.wikimedia.org/T380236" class="extiw" title="phab:T380236">T380236</a></li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>20:50 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:40 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:40 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:32 sukhe@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host cp7007.magru.wmnet with OS bullseye</li> <li>20:29 sukhe@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>20:24 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2041.codfw.wmnet with OS bookworm</li> <li>20:24 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:10 jhathaway@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>20:10 jhathaway@cumin1002: START - Cookbook sre.hosts.downtime for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>20:05 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>20:03 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-worker1183.eqiad.wmnet with OS bullseye</li> <li>20:03 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>19:47 pt1979@cumin2002: END (FAIL) - Cookbook sre.hosts.dhcp (exit_code=99) for host cp7007.magru.wmnet</li> <li>19:41 sukhe@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cp7007.magru.wmnet with OS bullseye</li> <li>19:40 pt1979@cumin2002: START - Cookbook sre.hosts.dhcp for host cp7007.magru.wmnet</li> <li>19:34 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>19:17 ebernhardson@deploy2002: Finished deploy [airflow-dags/search@a4d0954]: mjolnir: <a href="https://phabricator.wikimedia.org/T379045" class="extiw" title="phab:T379045">T379045</a> Increase maxResultSize (duration: 00m 26s)</li> <li>19:16 ebernhardson@deploy2002: Started deploy [airflow-dags/search@a4d0954]: mjolnir: <a href="https://phabricator.wikimedia.org/T379045" class="extiw" title="phab:T379045">T379045</a> Increase maxResultSize</li> <li>19:15 sukhe@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>19:14 sukhe@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host cp7007.magru.wmnet with OS bullseye</li> <li>19:12 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-worker1183.eqiad.wmnet with reason: host reimage</li> <li>19:08 sukhe@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>19:08 sukhe@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host cp7007.magru.wmnet with OS bullseye</li> <li>19:08 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on an-worker1183.eqiad.wmnet with reason: host reimage</li> <li>19:05 jhathaway@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>19:05 jhathaway@cumin1002: START - Cookbook sre.hosts.downtime for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>18:53 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host an-worker1183.eqiad.wmnet with OS bullseye</li> <li>18:53 brett: Import ncmonitor 1.3.0-1 into main apt repo</li> <li>18:52 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host an-worker1183.eqiad.wmnet with OS bullseye</li> <li>18:48 sukhe@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>18:47 sukhe@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cp7007.magru.wmnet with OS bullseye</li> <li>18:39 amastilovic@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:36 amastilovic@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:34 amastilovic@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:34 sukhe@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>18:34 amastilovic@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:34 sukhe@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cp7007.magru.wmnet with OS bullseye</li> <li>18:32 jhathaway@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>18:32 jhathaway@cumin1002: START - Cookbook sre.hosts.downtime for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>18:07 sukhe@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>17:57 brennen@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092875" class="extiw" title="gerrit:1092875">Prevent ce_event_wikis query when feature flag is off (T380288)</a> (duration: 15m 10s)</li> <li>17:56 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1326.eqiad.wmnet with OS bookworm</li> <li>17:56 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:55 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:54 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1327.eqiad.wmnet with OS bookworm</li> <li>17:53 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:53 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:52 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host an-worker1183.eqiad.wmnet with OS bullseye</li> <li>17:50 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1325.eqiad.wmnet with OS bookworm</li> <li>17:50 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:50 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:50 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host an-worker1183.eqiad.wmnet with OS bullseye</li> <li>17:50 brennen@deploy2002: daimona, brennen: Continuing with sync</li> <li>17:48 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1323.eqiad.wmnet with OS bookworm</li> <li>17:48 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:47 cmooney@cumin1002: END (FAIL) - Cookbook sre.network.configure-switch-interfaces (exit_code=99) for host wikikube-worker1290</li> <li>17:47 cmooney@cumin1002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker1290</li> <li>17:47 brennen@deploy2002: daimona, brennen: Backport for <a href="https://gerrit.wikimedia.org/r/1092875" class="extiw" title="gerrit:1092875">Prevent ce_event_wikis query when feature flag is off (T380288)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>17:47 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:45 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1322.eqiad.wmnet with OS bookworm</li> <li>17:45 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:43 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:42 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on wikikube-worker1290.eqiad.wmnet with reason: being moved to new port</li> <li>17:42 cmooney@cumin1002: START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on wikikube-worker1290.eqiad.wmnet with reason: being moved to new port</li> <li>17:42 jhathaway@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>17:41 brennen@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092875" class="extiw" title="gerrit:1092875">Prevent ce_event_wikis query when feature flag is off (T380288)</a></li> <li>17:41 jhathaway@cumin1002: START - Cookbook sre.hosts.downtime for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>17:41 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1324.eqiad.wmnet with OS bookworm</li> <li>17:41 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:40 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:38 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1326.eqiad.wmnet with reason: host reimage</li> <li>17:37 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2110.codfw.wmnet with OS bullseye</li> <li>17:37 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:37 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:36 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1327.eqiad.wmnet with reason: host reimage</li> <li>17:34 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host an-worker1183.eqiad.wmnet with OS bullseye</li> <li>17:32 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1325.eqiad.wmnet with reason: host reimage</li> <li>17:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1323.eqiad.wmnet with reason: host reimage</li> <li>17:28 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1326.eqiad.wmnet with reason: host reimage</li> <li>17:28 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1327.eqiad.wmnet with reason: host reimage</li> <li>17:28 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1325.eqiad.wmnet with reason: host reimage</li> <li>17:26 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1322.eqiad.wmnet with reason: host reimage</li> <li>17:23 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1324.eqiad.wmnet with reason: host reimage</li> <li>17:19 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2110.codfw.wmnet with reason: host reimage</li> <li>17:18 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1323.eqiad.wmnet with reason: host reimage</li> <li>17:18 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1314.eqiad.wmnet with OS bookworm</li> <li>17:18 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:18 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1324.eqiad.wmnet with reason: host reimage</li> <li>17:18 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1322.eqiad.wmnet with reason: host reimage</li> <li>17:18 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:16 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2110.codfw.wmnet with reason: host reimage</li> <li>17:15 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>17:15 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1318.eqiad.wmnet with OS bookworm</li> <li>17:15 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:14 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:11 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1319.eqiad.wmnet with OS bookworm</li> <li>17:11 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:11 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:11 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1326.eqiad.wmnet with OS bookworm</li> <li>17:10 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1327.eqiad.wmnet with OS bookworm</li> <li>17:10 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1325.eqiad.wmnet with OS bookworm</li> <li>17:09 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1320.eqiad.wmnet with OS bookworm</li> <li>17:09 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:08 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:04 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1321.eqiad.wmnet with OS bookworm</li> <li>17:04 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:04 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:02 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1316.eqiad.wmnet with OS bookworm</li> <li>17:02 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:01 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>17:00 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1323.eqiad.wmnet with OS bookworm</li> <li>17:00 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1324.eqiad.wmnet with OS bookworm</li> <li>17:00 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1322.eqiad.wmnet with OS bookworm</li> <li>17:00 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2110.codfw.wmnet with OS bullseye</li> <li>17:00 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2110']</li> <li>17:00 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1314.eqiad.wmnet with reason: host reimage</li> <li>17:00 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2110']</li> <li>16:58 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1317.eqiad.wmnet with OS bookworm</li> <li>16:58 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>16:58 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>16:56 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1318.eqiad.wmnet with reason: host reimage</li> <li>16:56 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1315.eqiad.wmnet with OS bookworm</li> <li>16:56 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>16:55 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>16:53 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1319.eqiad.wmnet with reason: host reimage</li> <li>16:52 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1313.eqiad.wmnet with OS bookworm</li> <li>16:52 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>16:52 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>16:50 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1320.eqiad.wmnet with reason: host reimage</li> <li>16:46 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1321.eqiad.wmnet with reason: host reimage</li> <li>16:43 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1316.eqiad.wmnet with reason: host reimage</li> <li>16:41 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1317.eqiad.wmnet with reason: host reimage</li> <li>16:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:37 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1315.eqiad.wmnet with reason: host reimage</li> <li>16:36 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1320.eqiad.wmnet with reason: host reimage</li> <li>16:36 fabfur@cumin1002: conftool action : set/pooled=yes; selector: name=cp7007.magru.wmnet</li> <li>16:35 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1321.eqiad.wmnet with reason: host reimage</li> <li>16:34 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1318.eqiad.wmnet with reason: host reimage</li> <li>16:34 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1319.eqiad.wmnet with reason: host reimage</li> <li>16:34 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1313.eqiad.wmnet with reason: host reimage</li> <li>16:33 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1316.eqiad.wmnet with reason: host reimage</li> <li>16:33 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1317.eqiad.wmnet with reason: host reimage</li> <li>16:33 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1315.eqiad.wmnet with reason: host reimage</li> <li>16:31 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1314.eqiad.wmnet with reason: host reimage</li> <li>16:30 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1313.eqiad.wmnet with reason: host reimage</li> <li>16:29 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:28 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:26 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:24 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2142.codfw.wmnet with OS bookworm</li> <li>16:19 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>16:17 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1319.eqiad.wmnet with OS bookworm</li> <li>16:17 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1320.eqiad.wmnet with OS bookworm</li> <li>16:17 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1321.eqiad.wmnet with OS bookworm</li> <li>16:17 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1318.eqiad.wmnet with OS bookworm</li> <li>16:16 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2141.codfw.wmnet with OS bookworm</li> <li>16:15 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1317.eqiad.wmnet with OS bookworm</li> <li>16:15 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1316.eqiad.wmnet with OS bookworm</li> <li>16:15 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1315.eqiad.wmnet with OS bookworm</li> <li>16:13 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1314.eqiad.wmnet with OS bookworm</li> <li>16:13 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1313.eqiad.wmnet with OS bookworm</li> <li>16:13 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2138.codfw.wmnet with OS bookworm</li> <li>16:09 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2137.codfw.wmnet with OS bookworm</li> <li>16:07 dreamyjazz@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092856" class="extiw" title="gerrit:1092856">ExperimentUserDefaultsManager: Decrease log severity to debug (T380271)</a> (duration: 13m 16s)</li> <li>16:04 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2142.codfw.wmnet with reason: host reimage</li> <li>16:03 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>16:00 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2139.codfw.wmnet with reason: host reimage</li> <li>15:59 dreamyjazz@deploy2002: dreamyjazz: Continuing with sync</li> <li>15:59 dreamyjazz@deploy2002: dreamyjazz: Backport for <a href="https://gerrit.wikimedia.org/r/1092856" class="extiw" title="gerrit:1092856">ExperimentUserDefaultsManager: Decrease log severity to debug (T380271)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:57 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2141.codfw.wmnet with reason: host reimage</li> <li>15:55 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:54 cgoubert@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:53 dreamyjazz@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092856" class="extiw" title="gerrit:1092856">ExperimentUserDefaultsManager: Decrease log severity to debug (T380271)</a></li> <li>15:53 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2138.codfw.wmnet with reason: host reimage</li> <li>15:50 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2137.codfw.wmnet with reason: host reimage</li> <li>15:48 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2142.codfw.wmnet with reason: host reimage</li> <li>15:47 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2141.codfw.wmnet with reason: host reimage</li> <li>15:47 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2139.codfw.wmnet with reason: host reimage</li> <li>15:46 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2138.codfw.wmnet with reason: host reimage</li> <li>15:46 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2137.codfw.wmnet with reason: host reimage</li> <li>15:45 moritzm: installing libheif security updates</li> <li>15:44 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2136.codfw.wmnet with reason: host reimage</li> <li>15:40 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2136.codfw.wmnet with reason: host reimage</li> <li>15:29 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2142.codfw.wmnet with OS bookworm</li> <li>15:29 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2141.codfw.wmnet with OS bookworm</li> <li>15:29 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>15:28 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2138.codfw.wmnet with OS bookworm</li> <li>15:28 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2137.codfw.wmnet with OS bookworm</li> <li>15:25 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>15:25 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2138.codfw.wmnet with OS bookworm</li> <li>15:22 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>15:21 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2142.codfw.wmnet with OS bookworm</li> <li>15:21 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2141.codfw.wmnet with OS bookworm</li> <li>15:21 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2137.codfw.wmnet with OS bookworm</li> <li>15:21 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>15:15 fabfur@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp7007.magru.wmnet with OS bullseye</li> <li>15:14 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_eqiad</li> <li>15:11 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_eqiad</li> <li>15:07 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>15:06 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>15:06 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>15:05 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>away: UTC afternoon deploys done</li> <li>14:59 tgr@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092333" class="extiw" title="gerrit:1092333">Use 'auth' rather than 'sso' as cookie prefix on the auth domain (T379811)</a> (duration: 14m 16s)</li> <li>14:52 tgr@deploy2002: tgr: Continuing with sync</li> <li>14:50 fabfur@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cp7007.magru.wmnet with reason: host reimage</li> <li>14:50 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>14:50 tgr@deploy2002: tgr: Backport for <a href="https://gerrit.wikimedia.org/r/1092333" class="extiw" title="gerrit:1092333">Use 'auth' rather than 'sso' as cookie prefix on the auth domain (T379811)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:49 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>14:49 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from eqiad to codfw</li> <li>14:48 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from eqiad to codfw</li> <li>14:46 fabfur@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on cp7007.magru.wmnet with reason: host reimage</li> <li>14:45 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>14:44 tgr@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092333" class="extiw" title="gerrit:1092333">Use 'auth' rather than 'sso' as cookie prefix on the auth domain (T379811)</a></li> <li>14:44 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2142.codfw.wmnet with OS bookworm</li> <li>14:44 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2141.codfw.wmnet with OS bookworm</li> <li>14:43 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>14:42 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2138.codfw.wmnet with OS bookworm</li> <li>14:41 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2137.codfw.wmnet with OS bookworm</li> <li>14:40 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>14:39 elukey: limit /v2/_catalog to internal IPs only for all Docker Registry nodes - <a href="https://phabricator.wikimedia.org/T378618" class="extiw" title="phab:T378618">T378618</a></li> <li>14:38 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092740" class="extiw" title="gerrit:1092740">Enable message group subscription feature for MediaWiki.org (T372386)</a> (duration: 16m 21s)</li> <li>14:35 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>14:34 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>14:34 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>14:33 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>14:31 kartik@deploy2002: kartik, abi: Continuing with sync</li> <li>14:31 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>14:30 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>14:29 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from eqiad to codfw</li> <li>14:28 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from eqiad to codfw</li> <li>14:28 kartik@deploy2002: kartik, abi: Backport for <a href="https://gerrit.wikimedia.org/r/1092740" class="extiw" title="gerrit:1092740">Enable message group subscription feature for MediaWiki.org (T372386)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:26 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_eqiad</li> <li>14:26 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_eqiad</li> <li>14:25 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>14:24 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>14:23 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>14:23 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>14:22 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092740" class="extiw" title="gerrit:1092740">Enable message group subscription feature for MediaWiki.org (T372386)</a></li> <li>14:22 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>14:21 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>14:21 fabfur@cumin1002: START - Cookbook sre.hosts.reimage for host cp7007.magru.wmnet with OS bullseye</li> <li>14:21 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_drmrs</li> <li>14:18 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_drmrs</li> <li>14:17 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092257" class="extiw" title="gerrit:1092257">Enable the Contribute menu in 3rd group of Wikis (T375301)</a> (duration: 15m 07s)</li> <li>14:15 joal@deploy2002: Finished deploy [analytics/refinery@295d5a4]: Regular analytics weekly train [analytics/refinery@295d5a44] (duration: 08m 56s)</li> <li>14:11 kartik@deploy2002: kartik: Continuing with sync</li> <li>14:10 akosiaris@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker1290.eqiad.wmnet</li> <li>14:10 kartik@deploy2002: kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1092257" class="extiw" title="gerrit:1092257">Enable the Contribute menu in 3rd group of Wikis (T375301)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:10 akosiaris@cumin1002: START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker1290.eqiad.wmnet</li> <li>14:07 ihurbain@deploy2002: helmfile [codfw] DONE helmfile.d/services/proton: apply</li> <li>14:06 joal@deploy2002: Started deploy [analytics/refinery@295d5a4]: Regular analytics weekly train [analytics/refinery@295d5a44]</li> <li>14:06 ihurbain@deploy2002: helmfile [codfw] START helmfile.d/services/proton: apply</li> <li>14:05 ihurbain@deploy2002: helmfile [eqiad] DONE helmfile.d/services/proton: apply</li> <li>14:04 ihurbain@deploy2002: helmfile [eqiad] START helmfile.d/services/proton: apply</li> <li>14:03 ihurbain@deploy2002: helmfile [staging] DONE helmfile.d/services/proton: apply</li> <li>14:02 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092257" class="extiw" title="gerrit:1092257">Enable the Contribute menu in 3rd group of Wikis (T375301)</a></li> <li>14:02 ihurbain@deploy2002: helmfile [staging] START helmfile.d/services/proton: apply</li> <li>14:01 ihurbain@deploy2002: helmfile [staging] DONE helmfile.d/services/proton: apply</li> <li>14:01 ihurbain@deploy2002: helmfile [staging] START helmfile.d/services/proton: apply</li> <li>13:27 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_drmrs</li> <li>13:27 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_drmrs</li> <li>13:08 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 266098</li> <li>13:08 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 266098</li> <li>13:08 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 267521</li> <li>13:07 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 267521</li> <li>13:07 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 201838</li> <li>13:06 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 201838</li> <li>13:06 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 262979</li> <li>13:06 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 262979</li> <li>13:06 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 266631</li> <li>13:06 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 266631</li> <li>13:05 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 53180</li> <li>13:05 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 53180</li> <li>13:05 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 21574</li> <li>13:05 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 21574</li> <li>12:57 cgoubert@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>12:55 cgoubert@cumin1002: START - Cookbook sre.dns.netbox</li> <li>12:43 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>12:42 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>12:41 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from eqiad to codfw</li> <li>12:40 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from eqiad to codfw</li> <li>12:38 arnaudb@cumin1002: END (FAIL) - Cookbook sre.switchdc.databases.prepare (exit_code=99) for the switch from eqiad to codfw</li> <li>12:36 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from eqiad to codfw</li> <li>12:35 moritzm: removing ganeti1016 from active Ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>12:30 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_codfw</li> <li>12:27 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_codfw</li> <li>12:23 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>12:22 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>12:20 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>12:18 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>11:59 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1016.eqiad.wmnet</li> <li>11:44 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 100%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71095">https://phabricator.wikimedia.org/P71095</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-114422-arnaudb.json</li> <li>11:40 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_codfw</li> <li>11:40 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_codfw</li> <li>11:29 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 75%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71094">https://phabricator.wikimedia.org/P71094</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-112917-arnaudb.json</li> <li>11:14 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 50%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71093">https://phabricator.wikimedia.org/P71093</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-111411-arnaudb.json</li> <li>11:05 jiji@cumin1002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host mc-gp2004.codfw.wmnet</li> <li>11:03 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 207947</li> <li>11:03 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'configure' for AS: 207947</li> <li>10:59 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 25%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71092">https://phabricator.wikimedia.org/P71092</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-105906-arnaudb.json</li> <li>10:58 jiji@cumin1002: START - Cookbook sre.hosts.reboot-single for host mc-gp2004.codfw.wmnet</li> <li>10:44 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 15%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71091">https://phabricator.wikimedia.org/P71091</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-104401-arnaudb.json</li> <li>10:41 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_eqsin</li> <li>10:37 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_eqsin</li> <li>10:28 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 10%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71090">https://phabricator.wikimedia.org/P71090</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-102855-arnaudb.json</li> <li>10:27 jmm@cumin2002: END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-docker-registry (exit_code=0) rolling restart_daemons on A:docker-registry</li> <li>10:25 jmm@cumin2002: START - Cookbook sre.misc-clusters.roll-restart-reboot-docker-registry rolling restart_daemons on A:docker-registry</li> <li>10:16 moritzm: restart spamd on vrts to pick up openssl updates</li> <li>10:13 arnaudb@cumin1002: dbctl commit (dc=all): 'db2216 (re)pooling @ 5%: repool', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71089">https://phabricator.wikimedia.org/P71089</a> and previous config saved to /var/cache/conftool/dbconfig/20241119-101350-arnaudb.json</li> <li>10:02 moritzm: installing openssl security updates</li> <li>10:00 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>10:00 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>09:59 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>09:59 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>09:58 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>09:58 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>09:55 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from eqiad to codfw</li> <li>09:52 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from eqiad to codfw</li> <li>09:51 dcausse@deploy2002: helmfile [staging] DONE helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:51 dcausse@deploy2002: helmfile [staging] START helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:49 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from eqiad to codfw</li> <li>09:49 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from eqiad to codfw</li> <li>09:42 fabfur: upgrade haproxy on cp-text|upload_eqsin (<a href="https://phabricator.wikimedia.org/T379891" class="extiw" title="phab:T379891">T379891</a>)</li> <li>09:42 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_eqsin</li> <li>09:41 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_eqsin</li> <li>09:39 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>09:39 dcausse@deploy2002: helmfile [staging] DONE helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:39 dcausse@deploy2002: helmfile [staging] START helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:39 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>09:39 dcausse@deploy2002: helmfile [eqiad] DONE helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:38 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>09:38 dcausse@deploy2002: helmfile [eqiad] START helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:35 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>09:33 dcausse@deploy2002: helmfile [codfw] DONE helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:32 dcausse@deploy2002: helmfile [codfw] START helmfile.d/services/cirrus-streaming-updater: apply</li> <li>09:19 aklapper@deploy2002: rebuilt and synchronized wikiversions files: group0 to 1.44.0-wmf.4 refs <a href="https://phabricator.wikimedia.org/T375663" class="extiw" title="phab:T375663">T375663</a></li> <li>09:18 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>09:18 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>08:59 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092752" class="extiw" title="gerrit:1092752">Add + to nowiki in core-Permissions.php (T380252)</a> (duration: 10m 17s)</li> <li>08:54 urbanecm@deploy2002: urbanecm, jhsoby: Continuing with sync</li> <li>08:54 urbanecm@deploy2002: urbanecm, jhsoby: Backport for <a href="https://gerrit.wikimedia.org/r/1092752" class="extiw" title="gerrit:1092752">Add + to nowiki in core-Permissions.php (T380252)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:49 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092752" class="extiw" title="gerrit:1092752">Add + to nowiki in core-Permissions.php (T380252)</a></li> <li>08:48 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092741" class="extiw" title="gerrit:1092741">fix tours by finishing partial variable rename (T380071)</a>, <a href="https://gerrit.wikimedia.org/r/1092364" class="extiw" title="gerrit:1092364">affcom contactpages: Fix Letter of intent and logo field labels (T375392)</a>, <a href="https://gerrit.wikimedia.org/r/1092743" class="extiw" title="gerrit:1092743">Add nowiki to commonsuploads dblist (T380252)</a> (duration: 14m 29s)</li> <li>08:43 urbanecm@deploy2002: ammarpad, migr, jhsoby, urbanecm: Continuing with sync</li> <li>08:39 urbanecm@deploy2002: ammarpad, migr, jhsoby, urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1092741" class="extiw" title="gerrit:1092741">fix tours by finishing partial variable rename (T380071)</a>, <a href="https://gerrit.wikimedia.org/r/1092364" class="extiw" title="gerrit:1092364">affcom contactpages: Fix Letter of intent and logo field labels (T375392)</a>, <a href="https://gerrit.wikimedia.org/r/1092743" class="extiw" title="gerrit:1092743">Add nowiki to commonsuploads dblist (T380252)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:34 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092741" class="extiw" title="gerrit:1092741">fix tours by finishing partial variable rename (T380071)</a>, <a href="https://gerrit.wikimedia.org/r/1092364" class="extiw" title="gerrit:1092364">affcom contactpages: Fix Letter of intent and logo field labels (T375392)</a>, <a href="https://gerrit.wikimedia.org/r/1092743" class="extiw" title="gerrit:1092743">Add nowiki to commonsuploads dblist (T380252)</a></li> <li>08:29 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1082726" class="extiw" title="gerrit:1082726">Translate Event Logging: Enable using $wgTranslateEnableEventLogging (T364460)</a>, <a href="https://gerrit.wikimedia.org/r/1092258" class="extiw" title="gerrit:1092258">CirrusSearch: enable offloading weighted tags via EventBus (T378983 T377150)</a>, <a href="https://gerrit.wikimedia.org/r/1091197" class="extiw" title="gerrit:1091197">[GrowthExperiments] Add virtual domain config (T354939)</a> (duration: 24m 42s)</li> <li>08:22 urbanecm@deploy2002: urbanecm, wangombe, pfischer: Continuing with sync</li> <li>08:12 urbanecm@deploy2002: urbanecm, wangombe, pfischer: Backport for <a href="https://gerrit.wikimedia.org/r/1082726" class="extiw" title="gerrit:1082726">Translate Event Logging: Enable using $wgTranslateEnableEventLogging (T364460)</a>, <a href="https://gerrit.wikimedia.org/r/1092258" class="extiw" title="gerrit:1092258">CirrusSearch: enable offloading weighted tags via EventBus (T378983 T377150)</a>, <a href="https://gerrit.wikimedia.org/r/1091197" class="extiw" title="gerrit:1091197">[GrowthExperiments] Add virtual domain config (T354939)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:04 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1082726" class="extiw" title="gerrit:1082726">Translate Event Logging: Enable using $wgTranslateEnableEventLogging (T364460)</a>, <a href="https://gerrit.wikimedia.org/r/1092258" class="extiw" title="gerrit:1092258">CirrusSearch: enable offloading weighted tags via EventBus (T378983 T377150)</a>, <a href="https://gerrit.wikimedia.org/r/1091197" class="extiw" title="gerrit:1091197">[GrowthExperiments] Add virtual domain config (T354939)</a></li> <li>07:45 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2202.codfw.wmnet with reason: sad</li> <li>07:45 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2202.codfw.wmnet with reason: sad</li> <li>07:41 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on db1246.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T374215" class="extiw" title="phab:T374215">T374215</a> - hw maintenance</li> <li>07:40 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on db1246.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T374215" class="extiw" title="phab:T374215">T374215</a> - hw maintenance</li> <li>07:32 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1016.eqiad.wmnet</li> <li>07:31 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1016.eqiad.wmnet</li> <li>07:24 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1016.eqiad.wmnet</li> <li>05:01 mwpresync@deploy2002: Pruned MediaWiki: 1.44.0-wmf.1 (duration: 01m 18s)</li> <li>04:52 mwpresync@deploy2002: Finished scap sync-world: testwikis to 1.44.0-wmf.4 refs <a href="https://phabricator.wikimedia.org/T375663" class="extiw" title="phab:T375663">T375663</a> (duration: 49m 01s)</li> <li>04:16 andrew@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudvirt1062.eqiad.wmnet with OS bookworm</li> <li>04:03 mwpresync@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.4 refs <a href="https://phabricator.wikimedia.org/T375663" class="extiw" title="phab:T375663">T375663</a></li> <li>04:00 ejegg: fundraising civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="463a12c5"><a class="external text" href="https://gerrit.wikimedia.org/r/q/463a12c5">463a12c5</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="e29243f0"><a class="external text" href="https://gerrit.wikimedia.org/r/q/e29243f0">e29243f0</a></span></li> <li>03:51 andrew@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1062.eqiad.wmnet with reason: host reimage</li> <li>03:48 andrew@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1062.eqiad.wmnet with reason: host reimage</li> <li>03:33 andrew@cumin1002: START - Cookbook sre.hosts.reimage for host cloudvirt1062.eqiad.wmnet with OS bookworm</li> <li>03:09 ejegg: payments-wiki upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="459f259b"><a class="external text" href="https://gerrit.wikimedia.org/r/q/459f259b">459f259b</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="c4463536"><a class="external text" href="https://gerrit.wikimedia.org/r/q/c4463536">c4463536</a></span></li> <li>02:31 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kafka-jumbo1018.eqiad.wmnet with OS bullseye</li> <li>02:30 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>02:30 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>02:23 ejegg: standalone (IPN listener) SmashPig upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="601405dc"><a class="external text" href="https://gerrit.wikimedia.org/r/q/601405dc">601405dc</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="131e92a5"><a class="external text" href="https://gerrit.wikimedia.org/r/q/131e92a5">131e92a5</a></span></li> <li>02:12 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kafka-jumbo1018.eqiad.wmnet with reason: host reimage</li> <li>02:08 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on kafka-jumbo1018.eqiad.wmnet with reason: host reimage</li> <li>01:54 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1018.eqiad.wmnet with OS bullseye</li> <li>01:54 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host kafka-jumbo1018.eqiad.wmnet with OS bullseye</li> <li>01:51 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kafka-jumbo1016.eqiad.wmnet with OS bullseye</li> <li>01:51 jclark@cumin1002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>01:50 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kafka-jumbo1017.eqiad.wmnet with OS bullseye</li> <li>01:50 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>01:40 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>01:24 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>01:24 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kafka-jumbo1017.eqiad.wmnet with reason: host reimage</li> <li>01:21 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on kafka-jumbo1017.eqiad.wmnet with reason: host reimage</li> <li>01:12 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host maps-test2006.codfw.wmnet with OS bookworm</li> <li>01:12 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>01:07 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1018.eqiad.wmnet with OS bullseye</li> <li>01:07 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1017.eqiad.wmnet with OS bullseye</li> <li>01:06 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host kafka-jumbo1017.eqiad.wmnet with OS bullseye</li> <li>01:03 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>01:02 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kafka-jumbo1016.eqiad.wmnet with reason: host reimage</li> <li>00:58 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on kafka-jumbo1016.eqiad.wmnet with reason: host reimage</li> <li>00:54 tzatziki: removing 1 file for legal compliance</li> <li>00:53 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>00:51 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host maps-test2005.codfw.wmnet with OS bookworm</li> <li>00:51 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>00:44 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1016.eqiad.wmnet with OS bullseye</li> <li>00:42 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on maps-test2006.codfw.wmnet with reason: host reimage</li> <li>00:41 tzatziki: removing 1 file for legal compliance</li> <li>00:39 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host kafka-jumbo1016.eqiad.wmnet with OS bullseye</li> <li>00:39 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on maps-test2006.codfw.wmnet with reason: host reimage</li> <li>00:34 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>00:18 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1017.eqiad.wmnet with OS bullseye</li> <li>00:18 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host kafka-jumbo1017.eqiad.wmnet with OS bullseye</li> <li>00:14 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host maps-test2006.codfw.wmnet with OS bookworm</li> <li>00:14 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on maps-test2005.codfw.wmnet with reason: host reimage</li> <li>00:14 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host maps-test2004.codfw.wmnet with OS bookworm</li> <li>00:14 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>00:10 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>00:10 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on maps-test2005.codfw.wmnet with reason: host reimage</li> <li>00:03 tzatziki: removing 1 file for legal compliance</li> <li>00:00 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host maps-test2003.codfw.wmnet with OS bookworm</li> <li>00:00 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-18" data-mw-thread-id="h-2024-11-18"><span data-mw-comment-start="" id="h-2024-11-18"></span>2024-11-18<span data-mw-comment-end="h-2024-11-18"></span></h2></div> <ul><li>23:51 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>23:50 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on maps-test2004.codfw.wmnet with reason: host reimage</li> <li>23:48 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on maps-test2004.codfw.wmnet with reason: host reimage</li> <li>23:46 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host maps-test2005.codfw.wmnet with OS bookworm</li> <li>23:32 tzatziki: removing 1 file for legal compliance</li> <li>23:31 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on maps-test2003.codfw.wmnet with reason: host reimage</li> <li>23:28 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host maps-test2002.codfw.wmnet with OS bookworm</li> <li>23:28 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>23:27 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>23:26 tzatziki: removing 1 file for legal compliance</li> <li>23:26 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on maps-test2003.codfw.wmnet with reason: host reimage</li> <li>23:25 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host maps-test2004.codfw.wmnet with OS bookworm</li> <li>23:19 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on thanos-be2005.codfw.wmnet with reason: host reimage</li> <li>23:15 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on thanos-be2005.codfw.wmnet with reason: host reimage</li> <li>23:12 tzatziki: removing 2 files for legal compliance</li> <li>23:09 eevans@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>23:09 eevans@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Additional IPs for Cassandra — restbase2036 - eevans@cumin1002"</li> <li>23:09 eevans@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Additional IPs for Cassandra — restbase2036 - eevans@cumin1002"</li> <li>23:08 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on maps-test2002.codfw.wmnet with reason: host reimage</li> <li>23:06 eevans@cumin1002: START - Cookbook sre.dns.netbox</li> <li>23:05 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on maps-test2002.codfw.wmnet with reason: host reimage</li> <li>23:04 eevans@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>23:04 eevans@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Additional IPs for Cassandra — restbase2036 - eevans@cumin1002"</li> <li>23:04 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host maps-test2003.codfw.wmnet with OS bookworm</li> <li>23:04 eevans@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Additional IPs for Cassandra — restbase2036 - eevans@cumin1002"</li> <li>23:03 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>23:01 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>23:00 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1018.eqiad.wmnet with OS bullseye</li> <li>23:00 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1017.eqiad.wmnet with OS bullseye</li> <li>23:00 eevans@cumin1002: START - Cookbook sre.dns.netbox</li> <li>22:59 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host kafka-jumbo1016.eqiad.wmnet with OS bullseye</li> <li>22:57 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>22:55 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2045.codfw.wmnet with OS bookworm</li> <li>22:55 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>22:55 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2044.codfw.wmnet with OS bookworm</li> <li>22:54 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2046.codfw.wmnet with OS bookworm</li> <li>22:54 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2043.codfw.wmnet with OS bookworm</li> <li>22:54 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2041.codfw.wmnet with OS bookworm</li> <li>22:52 tzatziki: removing 10 files for legal compliance</li> <li>22:50 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host maps-test2001.codfw.wmnet with OS bookworm</li> <li>22:50 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>22:49 bking@deploy2002: Finished deploy [wdqs/wdqs@9927a5a]: 0.3.150 (duration: 11m 59s)</li> <li>22:47 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>22:37 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host es2042.codfw.wmnet with OS bookworm</li> <li>22:37 bking@deploy2002: Started deploy [wdqs/wdqs@9927a5a]: 0.3.150</li> <li>22:22 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>22:18 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092336" class="extiw" title="gerrit:1092336">[GrowthExperiments] testwiki: Only enable Add Link for new accounts (T380204)</a> (duration: 09m 14s)</li> <li>22:13 urbanecm@deploy2002: urbanecm: Continuing with sync</li> <li>22:13 urbanecm@deploy2002: urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1092336" class="extiw" title="gerrit:1092336">[GrowthExperiments] testwiki: Only enable Add Link for new accounts (T380204)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:09 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092336" class="extiw" title="gerrit:1092336">[GrowthExperiments] testwiki: Only enable Add Link for new accounts (T380204)</a></li> <li>21:58 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092304" class="extiw" title="gerrit:1092304">Use WAN cache for JsonConfig remote fetch cache (T374746)</a>, <a href="https://gerrit.wikimedia.org/r/1092300" class="extiw" title="gerrit:1092300">Create no-link-recommendation variant (T377787 T380204)</a>, <a href="https://gerrit.wikimedia.org/r/1092295" class="extiw" title="gerrit:1092295">[GrowthExperiments] testwiki: Enable no-link-recommendation experiment (T380204)</a> (duration: 12m 10s)</li> <li>21:54 urbanecm@deploy2002: urbanecm, bvibber: Continuing with sync</li> <li>21:52 urbanecm@deploy2002: urbanecm, bvibber: Backport for <a href="https://gerrit.wikimedia.org/r/1092304" class="extiw" title="gerrit:1092304">Use WAN cache for JsonConfig remote fetch cache (T374746)</a>, <a href="https://gerrit.wikimedia.org/r/1092300" class="extiw" title="gerrit:1092300">Create no-link-recommendation variant (T377787 T380204)</a>, <a href="https://gerrit.wikimedia.org/r/1092295" class="extiw" title="gerrit:1092295">[GrowthExperiments] testwiki: Enable no-link-recommendation experiment (T380204)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:48 effie: upload prometheus-mcrouter-exporter_0.4.0+git20241118-1~wmf1 - <a href="https://phabricator.wikimedia.org/T380212" class="extiw" title="phab:T380212">T380212</a></li> <li>21:46 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092304" class="extiw" title="gerrit:1092304">Use WAN cache for JsonConfig remote fetch cache (T374746)</a>, <a href="https://gerrit.wikimedia.org/r/1092300" class="extiw" title="gerrit:1092300">Create no-link-recommendation variant (T377787 T380204)</a>, <a href="https://gerrit.wikimedia.org/r/1092295" class="extiw" title="gerrit:1092295">[GrowthExperiments] testwiki: Enable no-link-recommendation experiment (T380204)</a></li> <li>21:42 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002"</li> <li>21:36 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091839" class="extiw" title="gerrit:1091839">Rename everything referring to "SSO domain" to use "shared domain" (T379811)</a>, <a href="https://gerrit.wikimedia.org/r/1091841" class="extiw" title="gerrit:1091841">Rename shared domain sso.wikimedia.org to auth.wikimedia.org (T379811)</a>, <a href="https://gerrit.wikimedia.org/r/1091842" class="extiw" title="gerrit:1091842">Use DB name rather than server name in shared domain path prefix (T379811)</a> (duration: 10m 54s)</li> <li>21:31 urbanecm@deploy2002: matmarex, urbanecm: Continuing with sync</li> <li>21:30 urbanecm@deploy2002: matmarex, urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1091839" class="extiw" title="gerrit:1091839">Rename everything referring to "SSO domain" to use "shared domain" (T379811)</a>, <a href="https://gerrit.wikimedia.org/r/1091841" class="extiw" title="gerrit:1091841">Rename shared domain sso.wikimedia.org to auth.wikimedia.org (T379811)</a>, <a href="https://gerrit.wikimedia.org/r/1091842" class="extiw" title="gerrit:1091842">Use DB name rather than server name in shared domain path prefix (T379811)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:29 urbanecm: Add bvibber to wmf-deployment Gerrit group (existing deployer)</li> <li>21:26 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091839" class="extiw" title="gerrit:1091839">Rename everything referring to "SSO domain" to use "shared domain" (T379811)</a>, <a href="https://gerrit.wikimedia.org/r/1091841" class="extiw" title="gerrit:1091841">Rename shared domain sso.wikimedia.org to auth.wikimedia.org (T379811)</a>, <a href="https://gerrit.wikimedia.org/r/1091842" class="extiw" title="gerrit:1091842">Use DB name rather than server name in shared domain path prefix (T379811)</a></li> <li>21:21 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on maps-test2001.codfw.wmnet with reason: host reimage</li> <li>21:18 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on maps-test2001.codfw.wmnet with reason: host reimage</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2046.codfw.wmnet with OS bookworm</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2045.codfw.wmnet with OS bookworm</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2044.codfw.wmnet with OS bookworm</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2043.codfw.wmnet with OS bookworm</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2042.codfw.wmnet with OS bookworm</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host es2041.codfw.wmnet with OS bookworm</li> <li>21:16 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host maps-test2002.codfw.wmnet with OS bookworm</li> <li>21:15 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['es2042']</li> <li>21:15 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['es2042']</li> <li>21:15 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['es2041']</li> <li>21:15 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['es2041']</li> <li>21:14 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2042.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:11 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2044.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:11 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2045.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:10 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2043.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:10 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2041.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:03 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>21:01 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>21:01 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es2046.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:52 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bookworm</li> <li>20:51 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:51 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:49 jhathaway: disabling auto-reboot on re-imaging for debugging</li> <li>20:49 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host maps-test2001.codfw.wmnet with OS bookworm</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2046.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2045.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2044.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2043.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2042.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host es2041.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:39 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:37 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:37 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding es2041 to codfw - jhancock@cumin2002"</li> <li>20:37 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding es2041 to codfw - jhancock@cumin2002"</li> <li>20:33 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>20:29 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>20:23 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host restbase2037.codfw.wmnet with OS bullseye</li> <li>20:23 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:19 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:19 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2112.codfw.wmnet with OS bullseye</li> <li>20:19 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:14 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:12 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2113.codfw.wmnet with OS bullseye</li> <li>20:12 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:11 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:00 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on restbase2037.codfw.wmnet with reason: host reimage</li> <li>19:57 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on restbase2037.codfw.wmnet with reason: host reimage</li> <li>19:57 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2112.codfw.wmnet with reason: host reimage</li> <li>19:56 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>19:56 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:55 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>19:55 ebernhardson@deploy2002: Finished deploy [airflow-dags/search@594d3b5]: <a href="https://phabricator.wikimedia.org/T377153" class="extiw" title="phab:T377153">T377153</a> Release glent 0.3.5 (duration: 00m 27s)</li> <li>19:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2113.codfw.wmnet with reason: host reimage</li> <li>19:54 ebernhardson@deploy2002: Started deploy [airflow-dags/search@594d3b5]: <a href="https://phabricator.wikimedia.org/T377153" class="extiw" title="phab:T377153">T377153</a> Release glent 0.3.5</li> <li>19:52 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2112.codfw.wmnet with reason: host reimage</li> <li>19:51 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2113.codfw.wmnet with reason: host reimage</li> <li>19:37 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2163.codfw.wmnet with reason: host reimage</li> <li>19:36 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2112.codfw.wmnet with OS bullseye</li> <li>19:35 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2113.codfw.wmnet with OS bullseye</li> <li>19:35 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host restbase2037.codfw.wmnet with OS bullseye</li> <li>19:34 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2163.codfw.wmnet with reason: host reimage</li> <li>19:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2113']</li> <li>19:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['restbase2037']</li> <li>19:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2113']</li> <li>19:32 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['restbase2037']</li> <li>19:29 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2113.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>19:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host restbase2037.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>19:22 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>19:18 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2113.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>19:18 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>19:18 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host restbase2037.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>19:17 swfrench@deploy2002: Finished scap sync-world: Test deployment after adding mwdebug-next check command - <a href="https://phabricator.wikimedia.org/T372604" class="extiw" title="phab:T372604">T372604</a> (duration: 01m 31s)</li> <li>19:15 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>19:15 swfrench@deploy2002: Started scap sync-world: Test deployment after adding mwdebug-next check command - <a href="https://phabricator.wikimedia.org/T372604" class="extiw" title="phab:T372604">T372604</a></li> <li>19:08 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:58 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:57 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>18:56 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>18:46 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/mw-debug: apply</li> <li>18:45 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/mw-debug: apply</li> <li>18:43 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>18:41 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>18:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host an-worker1183.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>18:27 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:17 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:15 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:15 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:14 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:13 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:12 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>18:09 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:08 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:04 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:03 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:03 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>18:01 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>17:53 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host thanos-be2005.codfw.wmnet with OS bullseye</li> <li>17:34 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>17:28 xcollazo@deploy2002: Finished deploy [airflow-dags/analytics@16a5867]: Deploy latest DAGs to analytics Airflow instance. <a href="https://phabricator.wikimedia.org/T368755" class="extiw" title="phab:T368755">T368755</a>. (duration: 02m 10s)</li> <li>17:25 xcollazo@deploy2002: Started deploy [airflow-dags/analytics@16a5867]: Deploy latest DAGs to analytics Airflow instance. <a href="https://phabricator.wikimedia.org/T368755" class="extiw" title="phab:T368755">T368755</a>.</li> <li>17:24 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/blunderbuss: apply</li> <li>16:55 pt1979@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:55 pt1979@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: set DNS for new maps-test nodes - pt1979@cumin2002"</li> <li>16:55 pt1979@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: set DNS for new maps-test nodes - pt1979@cumin2002"</li> <li>16:50 volans: installing spicerack v8.16.2 on cumin1002</li> <li>16:50 pt1979@cumin2002: START - Cookbook sre.dns.netbox</li> <li>16:38 volans: installing spicerack v8.16.2 on cumin2002</li> <li>16:34 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker[1305-1312].eqiad.wmnet</li> <li>16:34 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker[1305-1312].eqiad.wmnet</li> <li>16:34 volans: uploaded spicerack_8.16.2 to apt.wikimedia.org bullseye-wikimedia</li> <li>16:30 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1311.eqiad.wmnet with OS bookworm</li> <li>16:25 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1310.eqiad.wmnet with OS bookworm</li> <li>16:22 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1312.eqiad.wmnet with OS bookworm</li> <li>16:19 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1306.eqiad.wmnet with OS bookworm</li> <li>16:16 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1308.eqiad.wmnet with OS bookworm</li> <li>16:14 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1309.eqiad.wmnet with OS bookworm</li> <li>16:13 jiji@cumin1002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host mc-gp1005.eqiad.wmnet</li> <li>16:11 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1311.eqiad.wmnet with reason: host reimage</li> <li>16:10 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1307.eqiad.wmnet with OS bookworm</li> <li>16:08 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1305.eqiad.wmnet with OS bookworm</li> <li>16:07 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1310.eqiad.wmnet with reason: host reimage</li> <li>16:06 jiji@cumin1002: START - Cookbook sre.hosts.reboot-single for host mc-gp1005.eqiad.wmnet</li> <li>16:04 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1312.eqiad.wmnet with reason: host reimage</li> <li>16:01 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1306.eqiad.wmnet with reason: host reimage</li> <li>15:58 Lucas_WMDE: UTC afternoon backport+config window done</li> <li>15:58 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092259" class="extiw" title="gerrit:1092259">Unified dashboard: Add UI for page collection recommendations (T368718)</a> (duration: 27m 17s)</li> <li>15:58 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1308.eqiad.wmnet with reason: host reimage</li> <li>15:56 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1312.eqiad.wmnet with reason: host reimage</li> <li>15:55 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1311.eqiad.wmnet with reason: host reimage</li> <li>15:54 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1309.eqiad.wmnet with reason: host reimage</li> <li>15:51 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1307.eqiad.wmnet with reason: host reimage</li> <li>15:51 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1310.eqiad.wmnet with reason: host reimage</li> <li>15:50 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1309.eqiad.wmnet with reason: host reimage</li> <li>15:49 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1308.eqiad.wmnet with reason: host reimage</li> <li>15:49 lucaswerkmeister-wmde@deploy2002: sbisson, lucaswerkmeister-wmde: Continuing with sync</li> <li>15:48 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1305.eqiad.wmnet with reason: host reimage</li> <li>15:48 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1307.eqiad.wmnet with reason: host reimage</li> <li>15:46 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1306.eqiad.wmnet with reason: host reimage</li> <li>15:45 lucaswerkmeister-wmde@deploy2002: sbisson, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1092259" class="extiw" title="gerrit:1092259">Unified dashboard: Add UI for page collection recommendations (T368718)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:45 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1305.eqiad.wmnet with reason: host reimage</li> <li>15:36 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1312.eqiad.wmnet with OS bookworm</li> <li>15:36 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1311.eqiad.wmnet with OS bookworm</li> <li>15:31 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1310.eqiad.wmnet with OS bookworm</li> <li>15:31 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1309.eqiad.wmnet with OS bookworm</li> <li>15:31 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1092259" class="extiw" title="gerrit:1092259">Unified dashboard: Add UI for page collection recommendations (T368718)</a></li> <li>15:30 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1308.eqiad.wmnet with OS bookworm</li> <li>15:29 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1307.eqiad.wmnet with OS bookworm</li> <li>15:27 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1306.eqiad.wmnet with OS bookworm</li> <li>15:26 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1305.eqiad.wmnet with OS bookworm</li> <li>15:11 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091605" class="extiw" title="gerrit:1091605">Revert "Allow other input and changes to trigger searchsuggestions to update" (T379983)</a> (duration: 08m 14s)</li> <li>15:07 lucaswerkmeister-wmde@deploy2002: samtar, lucaswerkmeister-wmde: Continuing with sync</li> <li>15:06 lucaswerkmeister-wmde@deploy2002: samtar, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1091605" class="extiw" title="gerrit:1091605">Revert "Allow other input and changes to trigger searchsuggestions to update" (T379983)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:03 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091605" class="extiw" title="gerrit:1091605">Revert "Allow other input and changes to trigger searchsuggestions to update" (T379983)</a></li> <li>15:00 arnaudb@cumin1002: dbctl commit (dc=all): 'manual depool commit', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71077">https://phabricator.wikimedia.org/P71077</a> and previous config saved to /var/cache/conftool/dbconfig/20241118-150020-arnaudb.json</li> <li>14:59 arnaudb@cumin1002: dbctl commit (dc=all): 'manual repool commit', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71076">https://phabricator.wikimedia.org/P71076</a> and previous config saved to /var/cache/conftool/dbconfig/20241118-145946-arnaudb.json</li> <li>14:56 arnaudb@cumin1002: END (FAIL) - Cookbook sre.mysql.pool (exit_code=99) db2216 slowly with 10 steps - slow motion repool <a href="https://phabricator.wikimedia.org/T380131" class="extiw" title="phab:T380131">T380131</a></li> <li>14:56 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db2216 slowly with 10 steps - slow motion repool <a href="https://phabricator.wikimedia.org/T380131" class="extiw" title="phab:T380131">T380131</a></li> <li>14:52 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db2150 slowly with 10 steps - slow repool db2150 <a href="https://phabricator.wikimedia.org/T380117" class="extiw" title="phab:T380117">T380117</a></li> <li>14:32 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker[1305-1312].eqiad.wmnet</li> <li>14:28 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker[1305-1312].eqiad.wmnet</li> <li>14:16 claime: running homer 'cr*-eqiad' '<a href="https://phabricator.wikimedia.org/T379454" class="extiw" title="phab:T379454">T379454</a>'</li> <li>14:11 jiji@cumin1002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host mc-gp1004.eqiad.wmnet</li> <li>14:09 btullis@cumin1002: END (PASS) - Cookbook sre.presto.roll-restart-workers (exit_code=0) for Presto an-presto cluster: Roll restart of all Presto's jvm daemons.</li> <li>14:04 jiji@cumin1002: START - Cookbook sre.hosts.reboot-single for host mc-gp1004.eqiad.wmnet</li> <li>13:50 jelto@deploy2002: helmfile [codfw] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:49 jelto@deploy2002: helmfile [codfw] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:49 jelto@deploy2002: helmfile [eqiad] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:48 jelto@deploy2002: helmfile [eqiad] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:47 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:46 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:37 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:37 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:35 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:35 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:35 mvolz@deploy2002: helmfile [codfw] DONE helmfile.d/services/citoid: apply</li> <li>13:34 mvolz@deploy2002: helmfile [codfw] START helmfile.d/services/citoid: apply</li> <li>13:34 mvolz@deploy2002: helmfile [eqiad] DONE helmfile.d/services/citoid: apply</li> <li>13:33 mvolz@deploy2002: helmfile [eqiad] START helmfile.d/services/citoid: apply</li> <li>13:31 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:31 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:31 mvolz@deploy2002: helmfile [staging] DONE helmfile.d/services/citoid: apply</li> <li>13:30 mvolz@deploy2002: helmfile [staging] START helmfile.d/services/citoid: apply</li> <li>13:28 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:28 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:27 btullis@cumin1002: START - Cookbook sre.presto.roll-restart-workers for Presto an-presto cluster: Roll restart of all Presto's jvm daemons.</li> <li>13:26 topranks: stopping netbox service on netbox-next test server to restore new database backup from production</li> <li>13:25 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:25 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:20 btullis@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-presto1018.eqiad.wmnet with OS bullseye</li> <li>13:16 urbanecm: mwmaint2002: Run `extensions/GrowthExperiments/maintenance/refreshLinkRecommendations.php` at `testwiki` for a bunch of pages (P71064 is list of commands executed; <a href="https://phabricator.wikimedia.org/T378983" class="extiw" title="phab:T378983">T378983</a>)</li> <li>13:04 jelto@deploy2002: helmfile [staging] DONE helmfile.d/services/wikidata-query-gui: apply</li> <li>13:03 jelto@deploy2002: helmfile [staging] START helmfile.d/services/wikidata-query-gui: apply</li> <li>13:01 moritzm: removing ganeti1021 from active Ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>12:56 btullis@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-presto1018.eqiad.wmnet with reason: host reimage</li> <li>12:54 btullis@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on an-presto1018.eqiad.wmnet with reason: host reimage</li> <li>12:39 btullis@cumin1002: START - Cookbook sre.hosts.reimage for host an-presto1018.eqiad.wmnet with OS bullseye</li> <li>12:38 btullis@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host an-presto1018.eqiad.wmnet with OS bullseye</li> <li>12:38 cgoubert@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>12:37 kart_: Updated recommendation api to 2024-11-13-183159-production (<a href="https://phabricator.wikimedia.org/T379592" class="extiw" title="phab:T379592">T379592</a>, <a href="https://phabricator.wikimedia.org/T379037" class="extiw" title="phab:T379037">T379037</a>)</li> <li>12:36 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db2150 slowly with 10 steps - slow repool db2150 <a href="https://phabricator.wikimedia.org/T380117" class="extiw" title="phab:T380117">T380117</a></li> <li>12:36 cgoubert@cumin1002: START - Cookbook sre.dns.netbox</li> <li>12:24 kartik@deploy2002: helmfile [ml-serve-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>12:22 btullis@cumin1002: START - Cookbook sre.hosts.reimage for host an-presto1018.eqiad.wmnet with OS bullseye</li> <li>12:22 kartik@deploy2002: helmfile [ml-serve-eqiad] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>12:21 btullis@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host an-presto1018.eqiad.wmnet with OS bullseye</li> <li>12:19 btullis@cumin1002: END (PASS) - Cookbook sre.druid.roll-restart-workers (exit_code=0) for Druid analytics cluster: Roll restart of Druid jvm daemons.</li> <li>12:15 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-product: apply</li> <li>12:14 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-product: apply</li> <li>12:13 fabfur@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-ulsfo</li> <li>12:13 kartik@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>12:10 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>12:09 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-product: apply</li> <li>12:08 btullis@cumin1002: START - Cookbook sre.hosts.reimage for host an-presto1018.eqiad.wmnet with OS bullseye</li> <li>12:02 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-product: apply</li> <li>12:00 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>11:59 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:59 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>11:58 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:58 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:47 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>11:45 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:45 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:41 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>11:41 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on db2216.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T380131" class="extiw" title="phab:T380131">T380131</a> - table corruption</li> <li>11:41 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on db2216.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T380131" class="extiw" title="phab:T380131">T380131</a> - table corruption</li> <li>11:41 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>11:41 urbanecm: mwmaint2002: Run `extensions/GrowthExperiments/maintenance/refreshLinkRecommendations.php` at `testwiki` for a bunch of pages (P71064 is list of commands executed; <a href="https://phabricator.wikimedia.org/T378983" class="extiw" title="phab:T378983">T378983</a>)</li> <li>11:33 btullis@cumin1002: START - Cookbook sre.druid.roll-restart-workers for Druid analytics cluster: Roll restart of Druid jvm daemons.</li> <li>11:25 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:25 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:21 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:16 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:50 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:50 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:50 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:49 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:46 dcausse@deploy2002: helmfile [eqiad] DONE helmfile.d/services/rdf-streaming-updater: apply</li> <li>10:46 dcausse@deploy2002: helmfile [eqiad] START helmfile.d/services/rdf-streaming-updater: apply</li> <li>10:45 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:45 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:43 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:43 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:41 dcausse@deploy2002: helmfile [codfw] DONE helmfile.d/services/rdf-streaming-updater: apply</li> <li>10:41 dcausse@deploy2002: helmfile [codfw] START helmfile.d/services/rdf-streaming-updater: apply</li> <li>10:39 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:37 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:27 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:27 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:15 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:14 fabfur: upgrade haproxy on cp-ulsfo (<a href="https://phabricator.wikimedia.org/T379891" class="extiw" title="phab:T379891">T379891</a>)</li> <li>10:14 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:14 fabfur@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-ulsfo</li> <li>10:13 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:13 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:47 dcausse@deploy2002: helmfile [staging] DONE helmfile.d/services/rdf-streaming-updater: apply</li> <li>09:47 dcausse@deploy2002: helmfile [staging] START helmfile.d/services/rdf-streaming-updater: apply</li> <li>09:42 moritzm: restarting nginx on acmechief hosts to pick up openssl updates</li> <li>09:24 moritzm: installing openssl security updates</li> <li>09:18 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:17 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:57 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091932" class="extiw" title="gerrit:1091932">Enable the Contribute menu in 2nd group of Wikis (T375300)</a> (duration: 11m 45s)</li> <li>08:55 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 40850</li> <li>08:55 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 40850</li> <li>08:53 kartik@deploy2002: kartik: Continuing with sync</li> <li>08:49 kartik@deploy2002: kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1091932" class="extiw" title="gerrit:1091932">Enable the Contribute menu in 2nd group of Wikis (T375300)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:45 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091932" class="extiw" title="gerrit:1091932">Enable the Contribute menu in 2nd group of Wikis (T375300)</a></li> <li>08:44 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:30:00 on registry1004.eqiad.wmnet with reason: testing</li> <li>08:44 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 0:30:00 on registry1004.eqiad.wmnet with reason: testing</li> <li>08:43 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091912" class="extiw" title="gerrit:1091912">bjnwikiquote: Add local logo (T375054)</a> (duration: 22m 55s)</li> <li>08:31 kartik@deploy2002: kartik, hamishz: Continuing with sync</li> <li>08:30 kartik@deploy2002: kartik, hamishz: Backport for <a href="https://gerrit.wikimedia.org/r/1091912" class="extiw" title="gerrit:1091912">bjnwikiquote: Add local logo (T375054)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:20 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091912" class="extiw" title="gerrit:1091912">bjnwikiquote: Add local logo (T375054)</a></li> <li>08:07 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>08:07 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>08:05 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>08:03 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>08:01 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>08:01 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>07:56 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1021.eqiad.wmnet</li> <li>07:54 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1020.eqiad.wmnet</li> <li>07:52 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1020.eqiad.wmnet</li> <li>07:51 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1020.eqiad.wmnet</li> <li>07:47 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1020.eqiad.wmnet</li> <li>07:46 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:46 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:46 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on pc1013.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T373037" class="extiw" title="phab:T373037">T373037</a>, host is not pooled</li> <li>07:46 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on pc1013.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T373037" class="extiw" title="phab:T373037">T373037</a>, host is not pooled</li> <li>06:31 kart_: Updated MinT to 2024-10-16-065051-production on eqiad</li> <li>06:28 kartik@deploy2002: helmfile [eqiad] DONE helmfile.d/services/machinetranslation: apply</li> <li>06:19 kartik@deploy2002: helmfile [eqiad] START helmfile.d/services/machinetranslation: apply</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-17" data-mw-thread-id="h-2024-11-17"><span data-mw-comment-start="" id="h-2024-11-17"></span>2024-11-17<span data-mw-comment-end="h-2024-11-17"></span></h2></div> <ul><li>16:41 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on db2216.codfw.wmnet with reason: Sad</li> <li>16:40 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 3 days, 0:00:00 on db2216.codfw.wmnet with reason: Sad</li> <li>16:35 ladsgroup@cumin1002: dbctl commit (dc=all): 'db2216 sad', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71059">https://phabricator.wikimedia.org/P71059</a> and previous config saved to /var/cache/conftool/dbconfig/20241117-163522-ladsgroup.json</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-16" data-mw-thread-id="h-2024-11-16"><span data-mw-comment-start="" id="h-2024-11-16"></span>2024-11-16<span data-mw-comment-end="h-2024-11-16"></span></h2></div> <ul><li>20:30 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host kafka-jumbo1017.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host kafka-jumbo1016.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host kafka-jumbo1018.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>18:09 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>18:09 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>18:08 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>18:06 jclark@cumin1002: START - Cookbook sre.hosts.provision for host an-worker1183.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>18:05 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:01 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:59 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:59 jclark@cumin1002: START - Cookbook sre.hosts.provision for host kafka-jumbo1018.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:56 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host kafka-jumbo1018.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:56 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:56 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>17:56 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>17:55 jclark@cumin1002: START - Cookbook sre.hosts.provision for host kafka-jumbo1016.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:55 jclark@cumin1002: START - Cookbook sre.hosts.provision for host kafka-jumbo1017.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:53 jclark@cumin1002: START - Cookbook sre.hosts.provision for host kafka-jumbo1018.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:52 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1313.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:52 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:50 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:50 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>17:50 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>17:45 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:14 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1323.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:11 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host wikikube-worker1327.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:11 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1327.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:09 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:09 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>17:09 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>17:08 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1313.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:05 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:05 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1327.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:01 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1326.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:57 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1321.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:55 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1324.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:54 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1322.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:54 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1320.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:53 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1325.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:52 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1319.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:52 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1316.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:51 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1318.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:50 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1315.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:49 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1317.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:49 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1314.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1326.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1327.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:36 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1323.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:36 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1324.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:36 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1322.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:36 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1321.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:36 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1320.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:35 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1325.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:32 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1318.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:32 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1317.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:32 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1316.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:31 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1315.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:31 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1314.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:31 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1319.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:30 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:30 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>16:30 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>16:27 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>00:44 tzatziki: removing 103 files for legal compliance</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-15" data-mw-thread-id="h-2024-11-15"><span data-mw-comment-start="" id="h-2024-11-15"></span>2024-11-15<span data-mw-comment-end="h-2024-11-15"></span></h2></div> <ul><li>23:42 tzatziki: removing 1 file for legal compliance</li> <li>23:19 tzatziki: removing 3 files for legal compliance</li> <li>22:34 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host elastic2112.codfw.wmnet with OS bullseye</li> <li>21:59 Dreamy_Jazz: Started MediaModeration scan on all wikis other than commonswiki attempting to scan all failed to be scanned images - <a class="external free" href="https://wikitech.wikimedia.org/wiki/MediaModeration">https://wikitech.wikimedia.org/wiki/MediaModeration</a></li> <li>21:59 Dreamy_Jazz: Started MediaModeration scan on commons wiki attempting to scan all failed to be scanned images - <a class="external free" href="https://wikitech.wikimedia.org/wiki/MediaModeration">https://wikitech.wikimedia.org/wiki/MediaModeration</a></li> <li>21:56 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2115.codfw.wmnet with OS bullseye</li> <li>21:56 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:56 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:53 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2114.codfw.wmnet with OS bullseye</li> <li>21:53 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:53 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:51 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2111.codfw.wmnet with OS bullseye</li> <li>21:50 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:50 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2115.codfw.wmnet with reason: host reimage</li> <li>21:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host restbase2038.codfw.wmnet with OS bullseye</li> <li>21:35 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2114.codfw.wmnet with reason: host reimage</li> <li>21:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host restbase2036.codfw.wmnet with OS bullseye</li> <li>21:35 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:33 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2111.codfw.wmnet with reason: host reimage</li> <li>21:30 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2115.codfw.wmnet with reason: host reimage</li> <li>21:30 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2114.codfw.wmnet with reason: host reimage</li> <li>21:30 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2111.codfw.wmnet with reason: host reimage</li> <li>21:28 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:14 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2115.codfw.wmnet with OS bullseye</li> <li>21:14 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2114.codfw.wmnet with OS bullseye</li> <li>21:14 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2112.codfw.wmnet with OS bullseye</li> <li>21:14 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host elastic2111.codfw.wmnet with OS bullseye</li> <li>21:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on restbase2038.codfw.wmnet with reason: host reimage</li> <li>21:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2115']</li> <li>21:13 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2115']</li> <li>21:12 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2114']</li> <li>21:12 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2114']</li> <li>21:12 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2112']</li> <li>21:12 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2112']</li> <li>21:12 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2111']</li> <li>21:12 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2111']</li> <li>21:11 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2110']</li> <li>21:11 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host elastic2113.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:10 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on restbase2036.codfw.wmnet with reason: host reimage</li> <li>21:08 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2114.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:08 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2111.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on restbase2038.codfw.wmnet with reason: host reimage</li> <li>21:07 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2115.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:07 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2112.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on restbase2036.codfw.wmnet with reason: host reimage</li> <li>21:04 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2115.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2114.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2113.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2112.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2111.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host elastic2110.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:54 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:54 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding elastic2110 to codfw - jhancock@cumin2002"</li> <li>20:54 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding elastic2110 to codfw - jhancock@cumin2002"</li> <li>20:50 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>20:45 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host restbase2038.codfw.wmnet with OS bullseye</li> <li>20:45 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host restbase2036.codfw.wmnet with OS bullseye</li> <li>20:44 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['restbase2036']</li> <li>20:44 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['restbase2038']</li> <li>20:43 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['restbase2038']</li> <li>20:43 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['restbase2036']</li> <li>20:43 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host restbase2038.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host restbase2036.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:41 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host restbase2037</li> <li>20:40 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host restbase2037</li> <li>20:40 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host restbase2037.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:32 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host restbase2038.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:32 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host restbase2037.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:32 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host restbase2036.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:31 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:31 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding restbase2036 to codfw - jhancock@cumin2002"</li> <li>20:31 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding restbase2036 to codfw - jhancock@cumin2002"</li> <li>20:27 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>19:54 dancy@deploy2002: Finished scap sync-world: Testing <a href="https://phabricator.wikimedia.org/T377883" class="extiw" title="phab:T377883">T377883</a> (duration: 03m 06s)</li> <li>19:51 dancy@deploy2002: Started scap sync-world: Testing <a href="https://phabricator.wikimedia.org/T377883" class="extiw" title="phab:T377883">T377883</a></li> <li>19:50 dancy@deploy2002: Installation of scap version "4.124.0" completed for 206 hosts</li> <li>19:46 dancy@deploy2002: Installing scap version "4.124.0" for 206 hosts</li> <li>18:53 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>18:52 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>18:35 cjming@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mpic: apply</li> <li>18:34 cjming@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mpic: apply</li> <li>18:32 cjming@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mpic-next: apply</li> <li>18:31 cjming@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mpic-next: apply</li> <li>18:15 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>18:15 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>18:09 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>18:08 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>16:58 mfossati@deploy2002: Finished deploy [airflow-dags/platform_eng@82083c4]: image suggestions hotfix - section titles denylist dependency (duration: 01m 58s)</li> <li>16:57 taavi: copy python3-flask-{keystone,oslolog} from bullseye-wikimedia to bookworm-wikimedia</li> <li>16:56 mfossati@deploy2002: Started deploy [airflow-dags/platform_eng@82083c4]: image suggestions hotfix - section titles denylist dependency</li> <li>16:27 herron@cumin2002: conftool action : set/pooled=yes; selector: name=aux-k8s-worker1005.eqiad.wmnet,cluster=aux-k8s,service=kubesvc</li> <li>16:27 herron@cumin2002: conftool action : set/weight=10; selector: name=aux-k8s-worker1005.eqiad.wmnet,cluster=aux-k8s,service=kubesvc</li> <li>16:22 herron@cumin2002: conftool action : set/pooled=yes; selector: name=aux-k8s-worker1004.eqiad.wmnet,cluster=aux-k8s,service=kubesvc</li> <li>16:22 herron@cumin2002: conftool action : set/weight=10; selector: name=aux-k8s-worker1004.eqiad.wmnet,cluster=aux-k8s,service=kubesvc</li> <li>16:09 sukhe@puppetserver1001: conftool action : set/pooled=yes; selector: name=cp4043.ulsfo.wmnet [reason: ATS fixed]</li> <li>16:08 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for cp4043.ulsfo.wmnet</li> <li>16:08 sukhe@cumin1002: START - Cookbook sre.hosts.remove-downtime for cp4043.ulsfo.wmnet</li> <li>16:06 sukhe@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-ats (exit_code=0) Rolling upgrade/restart of Apache Traffic Server on P{cp4051*} and A:cp for 9.2.6-1wm2</li> <li>16:03 sukhe@cumin1002: START - Cookbook sre.cdn.roll-upgrade-ats Rolling upgrade/restart of Apache Traffic Server on P{cp4051*} and A:cp for 9.2.6-1wm2</li> <li>16:00 sukhe: reprepro -C main include bullseye-wikimedia trafficserver_9.2.6-1wm2_amd64.changes: <a href="https://phabricator.wikimedia.org/T379797" class="extiw" title="phab:T379797">T379797</a></li> <li>15:47 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on db2230.codfw.wmnet,db1125.eqiad.wmnet with reason: testing stuff on test-s4</li> <li>15:47 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on db2230.codfw.wmnet,db1125.eqiad.wmnet with reason: testing stuff on test-s4</li> <li>15:42 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from eqiad to codfw</li> <li>15:41 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from eqiad to codfw</li> <li>15:40 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.finalize (exit_code=0) for the switch from codfw to eqiad</li> <li>15:39 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.finalize for the switch from codfw to eqiad</li> <li>15:39 arnaudb@cumin1002: END (PASS) - Cookbook sre.switchdc.databases.prepare (exit_code=0) for the switch from codfw to eqiad</li> <li>15:38 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-platform-eng: apply</li> <li>15:38 arnaudb@cumin1002: START - Cookbook sre.switchdc.databases.prepare for the switch from codfw to eqiad</li> <li>15:37 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-platform-eng: apply</li> <li>15:35 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>15:34 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>13:59 ayounsi@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>13:59 ayounsi@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Remove e8 lo0 IP - ayounsi@cumin1002"</li> <li>13:59 ayounsi@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Remove e8 lo0 IP - ayounsi@cumin1002"</li> <li>13:55 ayounsi@cumin1002: START - Cookbook sre.dns.netbox</li> <li>13:55 ayounsi@cumin1002: END (FAIL) - Cookbook sre.dns.netbox (exit_code=99)</li> <li>13:52 ayounsi@cumin1002: START - Cookbook sre.dns.netbox</li> <li>13:41 XioNoX: test no-passwords on mr1-eqsin - <a href="https://phabricator.wikimedia.org/T379464" class="extiw" title="phab:T379464">T379464</a></li> <li>13:31 ayounsi@cumin1002: END (FAIL) - Cookbook sre.hosts.decommission (exit_code=1) for hosts sretest1004.eqiad.wmnet</li> <li>13:31 ayounsi@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>13:31 ayounsi@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: sretest1004.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - ayounsi@cumin1002"</li> <li>13:31 ayounsi@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: sretest1004.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - ayounsi@cumin1002"</li> <li>13:27 ayounsi@cumin1002: START - Cookbook sre.dns.netbox</li> <li>13:24 cmooney@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) homer to cumin2002.codfw.wmnet,cumin1002.eqiad.wmnet with reason: Update homer wmf-plugin to export Netbox ipsec data - cmooney@cumin1002</li> <li>13:23 ayounsi@cumin1002: START - Cookbook sre.hosts.decommission for hosts sretest1004.eqiad.wmnet</li> <li>13:21 cmooney@cumin1002: START - Cookbook sre.deploy.python-code homer to cumin2002.codfw.wmnet,cumin1002.eqiad.wmnet with reason: Update homer wmf-plugin to export Netbox ipsec data - cmooney@cumin1002</li> <li>13:19 cmooney@cumin1002: END (FAIL) - Cookbook sre.deploy.python-code (exit_code=99) homer to cumin2002.codfw.wmnet,cumin1002.eqiad.wmnet with reason: Update homer wmf-plugin to export Netbox ipsec data - cmooney@cumin1002</li> <li>13:17 cmooney@cumin1002: START - Cookbook sre.deploy.python-code homer to cumin2002.codfw.wmnet,cumin1002.eqiad.wmnet with reason: Update homer wmf-plugin to export Netbox ipsec data - cmooney@cumin1002</li> <li>13:01 moritzm: imported 8u432-b06-2~deb12u1 to component/jdk8 for bookworm (forward port of the latest Java 8 security fixes for Bookworm)</li> <li>12:54 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host build2002.codfw.wmnet</li> <li>12:54 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host build2002.codfw.wmnet with OS bookworm</li> <li>12:36 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on build2002.codfw.wmnet with reason: host reimage</li> <li>12:32 jmm@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on build2002.codfw.wmnet with reason: host reimage</li> <li>12:27 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics: apply</li> <li>12:26 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics: apply</li> <li>12:19 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics: apply</li> <li>12:18 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>12:17 jmm@cumin2002: START - Cookbook sre.hosts.reimage for host build2002.codfw.wmnet with OS bookworm</li> <li>12:17 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>12:16 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM build2002.codfw.wmnet - jmm@cumin2002"</li> <li>12:15 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM build2002.codfw.wmnet - jmm@cumin2002"</li> <li>12:15 jmm@cumin2002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) build2002.codfw.wmnet on all recursors</li> <li>12:15 jmm@cumin2002: START - Cookbook sre.dns.wipe-cache build2002.codfw.wmnet on all recursors</li> <li>12:15 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>12:15 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM build2002.codfw.wmnet - jmm@cumin2002"</li> <li>12:11 cmooney@cumin1002: END (FAIL) - Cookbook sre.netbox.update-extras (exit_code=1) rolling restart_daemons on A:netbox</li> <li>12:11 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM build2002.codfw.wmnet - jmm@cumin2002"</li> <li>12:08 aokoth@cumin1002: END (PASS) - Cookbook sre.gitlab.upgrade (exit_code=0) on GitLab host gitlab2002.wikimedia.org with reason: Security Update</li> <li>12:03 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>12:03 jmm@cumin2002: START - Cookbook sre.ganeti.makevm for new host build2002.codfw.wmnet</li> <li>12:01 cmooney@cumin1002: START - Cookbook sre.netbox.update-extras rolling restart_daemons on A:netbox</li> <li>12:01 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.resource-report (exit_code=0)</li> <li>12:01 jmm@cumin2002: START - Cookbook sre.ganeti.resource-report</li> <li>12:00 cmooney@cumin1002: END (PASS) - Cookbook sre.netbox.update-extras (exit_code=0) rolling restart_daemons on A:netbox-canary</li> <li>11:58 cmooney@cumin1002: START - Cookbook sre.netbox.update-extras rolling restart_daemons on A:netbox-canary</li> <li>11:38 mfossati@deploy2002: Finished deploy [airflow-dags/platform_eng@2c533d6]: hotfix image suggestions weekly snapshots (duration: 00m 57s)</li> <li>11:37 mfossati@deploy2002: Started deploy [airflow-dags/platform_eng@2c533d6]: hotfix image suggestions weekly snapshots</li> <li>11:27 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>11:24 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker[1305-1312].eqiad.wmnet</li> <li>11:24 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker[1305-1312].eqiad.wmnet</li> <li>11:22 claime: homer 'lsw1-f5-eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>11:22 claime: homer 'lsw1-f6-eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>11:22 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>11:21 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>11:21 claime: homer 'lsw1-f7-eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>11:21 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>11:20 claime: homer 'lsw1-e7-eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>11:20 claime: homer 'lsw1-e6-eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>11:19 claime: homer 'lsw1-e5-eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>11:15 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:14 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:12 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:12 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:06 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:06 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:05 claime: homer 'cr*eqiad*' commit '<a href="https://phabricator.wikimedia.org/T377022" class="extiw" title="phab:T377022">T377022</a>'</li> <li>10:36 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:36 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:36 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:34 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on pc1013.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T373037" class="extiw" title="phab:T373037">T373037</a>, host is not pooled</li> <li>09:34 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on pc1013.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T373037" class="extiw" title="phab:T373037">T373037</a>, host is not pooled</li> <li>09:31 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:28 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:28 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:28 elukey@cumin2002: END (ERROR) - Cookbook sre.hosts.provision (exit_code=97) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:27 elukey@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:23 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:23 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:22 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:21 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:15 aokoth@cumin1002: START - Cookbook sre.gitlab.upgrade on GitLab host gitlab2002.wikimedia.org with reason: Security Update</li> <li>08:48 moritzm: installing Linux 6.1.115 kernel updates from Bookworm point release</li> <li>04:54 rzl@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 12:00:00 on db1246.eqiad.wmnet with reason: depooled</li> <li>04:54 rzl@cumin2002: START - Cookbook sre.hosts.downtime for 3 days, 12:00:00 on db1246.eqiad.wmnet with reason: depooled</li> <li>04:51 rzl@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 12:00:00 on db1246.eqiad.wmnet with reason: depooled</li> <li>04:50 rzl@cumin2002: START - Cookbook sre.hosts.downtime for 1 day, 12:00:00 on db1246.eqiad.wmnet with reason: depooled</li> <li>04:47 rzl@cumin2002: dbctl commit (dc=all): 'db1246 depooled', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71052">https://phabricator.wikimedia.org/P71052</a> and previous config saved to /var/cache/conftool/dbconfig/20241115-044705-rzl.json</li> <li>03:44 ejegg: fundraising python tools upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="c6e2dbcc"><a class="external text" href="https://gerrit.wikimedia.org/r/q/c6e2dbcc">c6e2dbcc</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="b230f718"><a class="external text" href="https://gerrit.wikimedia.org/r/q/b230f718">b230f718</a></span></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-14" data-mw-thread-id="h-2024-11-14"><span data-mw-comment-start="" id="h-2024-11-14"></span>2024-11-14<span data-mw-comment-end="h-2024-11-14"></span></h2></div> <ul><li>23:17 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="2a53f697"><a class="external text" href="https://gerrit.wikimedia.org/r/q/2a53f697">2a53f697</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="d49a064d"><a class="external text" href="https://gerrit.wikimedia.org/r/q/d49a064d">d49a064d</a></span></li> <li>22:59 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="2ab8334a"><a class="external text" href="https://gerrit.wikimedia.org/r/q/2ab8334a">2ab8334a</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="2a53f697"><a class="external text" href="https://gerrit.wikimedia.org/r/q/2a53f697">2a53f697</a></span></li> <li>22:37 brett@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on cp4043.ulsfo.wmnet with reason: ATS upgrade 9.2.6</li> <li>22:37 brett@cumin2002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on cp4043.ulsfo.wmnet with reason: ATS upgrade 9.2.6</li> <li>22:30 ryankemper: <a href="https://phabricator.wikimedia.org/T376150" class="extiw" title="phab:T376150">T376150</a> Depooled `wdqs20[18-20]` in preparation of merging <a class="external free" href="https://gerrit.wikimedia.org/r/c/operations/puppet/+/1088185">https://gerrit.wikimedia.org/r/c/operations/puppet/+/1088185</a></li> <li>21:49 aqu@deploy2002: Finished deploy [airflow-dags/analytics@7a66849]: Stage Refine: fix Airflow skip (duration: 00m 59s)</li> <li>21:48 aqu@deploy2002: Started deploy [airflow-dags/analytics@7a66849]: Stage Refine: fix Airflow skip</li> <li>21:47 aqu@deploy2002: Finished deploy [airflow-dags/analytics_test@7a66849]: Stage Refine: fix Airflow skip (duration: 00m 14s)</li> <li>21:47 aqu@deploy2002: Started deploy [airflow-dags/analytics_test@7a66849]: Stage Refine: fix Airflow skip</li> <li>21:26 aqu@deploy2002: Finished deploy [airflow-dags/analytics_test@2220747]: Stage Refine test fix (duration: 00m 16s)</li> <li>21:26 aqu@deploy2002: Started deploy [airflow-dags/analytics_test@2220747]: Stage Refine test fix</li> <li>21:20 cjming: end of UTC late backport window</li> <li>21:17 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1082853" class="extiw" title="gerrit:1082853">Redirect to wikis using subpages rather than namespaces too (T376923)</a> (duration: 13m 44s)</li> <li>21:13 cjming@deploy2002: cjming, pppery: Continuing with sync</li> <li>21:08 cjming@deploy2002: cjming, pppery: Backport for <a href="https://gerrit.wikimedia.org/r/1082853" class="extiw" title="gerrit:1082853">Redirect to wikis using subpages rather than namespaces too (T376923)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:04 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1082853" class="extiw" title="gerrit:1082853">Redirect to wikis using subpages rather than namespaces too (T376923)</a></li> <li>20:47 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>20:47 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>20:38 bvibber@deploy2002: helmfile [codfw] DONE helmfile.d/services/chart-renderer: apply</li> <li>20:37 bvibber@deploy2002: helmfile [codfw] START helmfile.d/services/chart-renderer: apply</li> <li>20:37 bvibber@deploy2002: helmfile [eqiad] DONE helmfile.d/services/chart-renderer: apply</li> <li>20:36 bvibber@deploy2002: helmfile [eqiad] START helmfile.d/services/chart-renderer: apply</li> <li>20:35 bvibber@deploy2002: helmfile [staging] DONE helmfile.d/services/chart-renderer: apply</li> <li>20:35 bvibber@deploy2002: helmfile [staging] START helmfile.d/services/chart-renderer: apply</li> <li>20:29 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0)</li> <li>20:28 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter</li> <li>20:24 bvibber@deploy2002: helmfile [codfw] DONE helmfile.d/services/chart-renderer: apply</li> <li>20:24 bvibber@deploy2002: helmfile [codfw] START helmfile.d/services/chart-renderer: apply</li> <li>20:24 bvibber@deploy2002: helmfile [eqiad] DONE helmfile.d/services/chart-renderer: apply</li> <li>20:24 bvibber@deploy2002: helmfile [eqiad] START helmfile.d/services/chart-renderer: apply</li> <li>20:23 bvibber@deploy2002: helmfile [staging] DONE helmfile.d/services/chart-renderer: apply</li> <li>20:23 bvibber@deploy2002: helmfile [staging] START helmfile.d/services/chart-renderer: apply</li> <li>20:23 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0) pool all active/active services in eqiad: Network maintenance complete - None</li> <li>20:01 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter pool all active/active services in eqiad: Network maintenance complete - None</li> <li>19:55 brennen@deploy2002: rebuilt and synchronized wikiversions files: group2 to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>19:40 eileen: tools upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="68f64e43"><a class="external text" href="https://gerrit.wikimedia.org/r/q/68f64e43">68f64e43</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="c6e2dbcc"><a class="external text" href="https://gerrit.wikimedia.org/r/q/c6e2dbcc">c6e2dbcc</a></span></li> <li>19:37 sukhe@cumin1002: END (PASS) - Cookbook sre.dns.admin (exit_code=0) DNS admin: pool site eqiad [reason: junos upgrade done, <a href="https://phabricator.wikimedia.org/T364092" class="extiw" title="phab:T364092">T364092</a>]</li> <li>19:37 sukhe@cumin1002: START - Cookbook sre.dns.admin DNS admin: pool site eqiad [reason: junos upgrade done, <a href="https://phabricator.wikimedia.org/T364092" class="extiw" title="phab:T364092">T364092</a>]</li> <li>19:20 James_F: Running `mwscript-k8s -f -- extensions/WikiLambda/maintenance/updateSecondaryTables.php --wiki=wikifunctionswiki --zType Z8 --report --verbose` for <a href="https://phabricator.wikimedia.org/T375972" class="extiw" title="phab:T375972">T375972</a>, <a href="https://phabricator.wikimedia.org/T367005" class="extiw" title="phab:T367005">T367005</a>, <a href="https://phabricator.wikimedia.org/T373038" class="extiw" title="phab:T373038">T373038</a>, <a href="https://phabricator.wikimedia.org/T358737" class="extiw" title="phab:T358737">T358737</a></li> <li>19:19 sukhe@cumin1002: END (PASS) - Cookbook sre.dns.roll-restart-ntp (exit_code=0) rolling restart_daemons on A:dnsbox</li> <li>19:14 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0)</li> <li>19:14 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter</li> <li>19:14 swfrench-wmf: running sre.discovery.datacenter status all to test deployed fix</li> <li>19:00 brennen: 1.44.0-wmf.3 train status (<a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a>): no current blockers, but holding for network maintenance.</li> <li>18:20 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1312.eqiad.wmnet with OS bullseye</li> <li>18:19 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0)</li> <li>18:18 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter</li> <li>18:16 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1310.eqiad.wmnet with OS bullseye</li> <li>18:13 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on cp4043.ulsfo.wmnet with reason: depooled, debugging</li> <li>18:13 sukhe@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on cp4043.ulsfo.wmnet with reason: depooled, debugging</li> <li>18:11 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:09 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1311.eqiad.wmnet with OS bullseye</li> <li>18:05 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1308.eqiad.wmnet with OS bullseye</li> <li>18:04 ladsgroup@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db1190 gradually with 4 steps - Maint over</li> <li>18:02 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1309.eqiad.wmnet with OS bullseye</li> <li>18:01 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1312.eqiad.wmnet with reason: host reimage</li> <li>17:59 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1307.eqiad.wmnet with OS bullseye</li> <li>17:57 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1310.eqiad.wmnet with reason: host reimage</li> <li>17:53 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2139.codfw.wmnet with reason: host reimage</li> <li>17:52 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1306.eqiad.wmnet with OS bullseye</li> <li>17:49 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1311.eqiad.wmnet with reason: host reimage</li> <li>17:46 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1308.eqiad.wmnet with reason: host reimage</li> <li>17:45 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1312.eqiad.wmnet with reason: host reimage</li> <li>17:45 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2139.codfw.wmnet with reason: host reimage</li> <li>17:44 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1311.eqiad.wmnet with reason: host reimage</li> <li>17:43 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1310.eqiad.wmnet with reason: host reimage</li> <li>17:42 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1309.eqiad.wmnet with reason: host reimage</li> <li>17:39 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1309.eqiad.wmnet with reason: host reimage</li> <li>17:39 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1307.eqiad.wmnet with reason: host reimage</li> <li>17:37 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1308.eqiad.wmnet with reason: host reimage</li> <li>17:37 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1307.eqiad.wmnet with reason: host reimage</li> <li>17:32 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1306.eqiad.wmnet with reason: host reimage</li> <li>17:29 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1306.eqiad.wmnet with reason: host reimage</li> <li>17:27 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>17:26 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1312.eqiad.wmnet with OS bullseye</li> <li>17:25 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1311.eqiad.wmnet with OS bullseye</li> <li>17:25 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1310.eqiad.wmnet with OS bullseye</li> <li>17:24 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0) status all services in all: None - None</li> <li>17:24 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter status all services in all: None - None</li> <li>17:21 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1309.eqiad.wmnet with OS bullseye</li> <li>17:19 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1308.eqiad.wmnet with OS bullseye</li> <li>17:19 ladsgroup@cumin1002: START - Cookbook sre.mysql.pool db1190 gradually with 4 steps - Maint over</li> <li>17:18 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0) depool all active/active services in eqiad: Network maintenance - None</li> <li>17:18 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1307.eqiad.wmnet with OS bullseye</li> <li>17:15 fabfur@cumin1002: conftool action : set/pooled=no; selector: name=4043.ulsfo.wmnet</li> <li>17:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2139.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:13 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>17:13 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>17:10 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1306.eqiad.wmnet with OS bullseye</li> <li>16:59 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1305.eqiad.wmnet with OS bullseye</li> <li>16:57 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter depool all active/active services in eqiad: Network maintenance - None</li> <li>16:52 mfossati@deploy2002: Finished deploy [airflow-dags/platform_eng@7c4873e]: decouple article-level image suggestions from section-level ones (duration: 00m 53s)</li> <li>16:51 mfossati@deploy2002: Started deploy [airflow-dags/platform_eng@7c4873e]: decouple article-level image suggestions from section-level ones</li> <li>16:45 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0) status all services in all: None - None</li> <li>16:45 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter status all services in all: None - None</li> <li>16:40 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1305.eqiad.wmnet with reason: host reimage</li> <li>16:38 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0)</li> <li>16:37 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter</li> <li>16:36 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1305.eqiad.wmnet with reason: host reimage</li> <li>16:36 swfrench@cumin2002: END (PASS) - Cookbook sre.discovery.datacenter (exit_code=0)</li> <li>16:36 swfrench@cumin2002: START - Cookbook sre.discovery.datacenter</li> <li>16:33 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on db1190.eqiad.wmnet with reason: Sad</li> <li>16:33 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 3 days, 0:00:00 on db1190.eqiad.wmnet with reason: Sad</li> <li>16:33 ladsgroup@cumin1002: dbctl commit (dc=all): 'db1190 sad', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71044">https://phabricator.wikimedia.org/P71044</a> and previous config saved to /var/cache/conftool/dbconfig/20241114-163317-ladsgroup.json</li> <li>16:31 klausman@deploy2002: helmfile [ml-staging-codfw] DONE helmfile.d/admin 'apply'.</li> <li>16:31 klausman@deploy2002: helmfile [ml-staging-codfw] START helmfile.d/admin 'apply'.</li> <li>16:18 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1305.eqiad.wmnet with OS bullseye</li> <li>16:04 cmooney@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 151575</li> <li>16:03 cmooney@cumin1002: START - Cookbook sre.network.peering with action 'configure' for AS: 151575</li> <li>16:01 papaul: ongoing maintenance on cr1-eqiad</li> <li>16:00 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2139.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:57 pt1979@cumin2002: END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on cr1-eqiad,cr1-eqiad IPV6,re0.cr1-eqiad.mgmt with reason: router upgrade</li> <li>15:57 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on cr1-eqiad,cr1-eqiad IPV6,re0.cr1-eqiad.mgmt with reason: router upgrade</li> <li>15:56 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on cp4043.ulsfo.wmnet with reason: depooled, debugging</li> <li>15:56 sukhe@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on cp4043.ulsfo.wmnet with reason: depooled, debugging</li> <li>15:55 pt1979@cumin2002: END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on cr1-eqiad,cr1-eqiad IPV6,cr1-eqiad.mgmt with reason: router upgrade</li> <li>15:55 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on cr1-eqiad,cr1-eqiad IPV6,cr1-eqiad.mgmt with reason: router upgrade</li> <li>15:49 moritzm: installing nss security updates</li> <li>15:48 reedy@deploy2002: Synchronized wmf-config/CommonSettings.php: <a href="https://phabricator.wikimedia.org/T379834" class="extiw" title="phab:T379834">T379834</a> (duration: 08m 02s)</li> <li>15:47 sukhe@puppetserver1001: conftool action : set/pooled=no; selector: name=cp4043.ulsfo.wmnet</li> <li>15:47 sukhe@cumin1002: END (ERROR) - Cookbook sre.cdn.roll-upgrade-ats (exit_code=97) Rolling upgrade/restart of Apache Traffic Server on P{cp4043*,cp4051*} and A:cp for 9.2.6-1wm1</li> <li>15:45 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for wikikube-ctrl2002.codfw.wmnet</li> <li>15:45 jayme@cumin2002: START - Cookbook sre.hosts.remove-downtime for wikikube-ctrl2002.codfw.wmnet</li> <li>15:45 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-ctrl2002.codfw.wmnet</li> <li>15:45 jayme@cumin2002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-ctrl2002.codfw.wmnet</li> <li>15:43 pt1979@cumin2002: END (PASS) - Cookbook sre.network.cf (exit_code=0)</li> <li>15:43 pt1979@cumin2002: START - Cookbook sre.network.cf</li> <li>15:42 sukhe@cumin1002: START - Cookbook sre.cdn.roll-upgrade-ats Rolling upgrade/restart of Apache Traffic Server on P{cp4043*,cp4051*} and A:cp for 9.2.6-1wm1</li> <li>15:40 stevemunene@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host an-presto1016.eqiad.wmnet with OS bullseye</li> <li>15:39 stevemunene@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>15:37 volans: installed spicerack v8.16.1 to cumin hosts</li> <li>15:36 sukhe@cumin1002: END (PASS) - Cookbook sre.dns.admin (exit_code=0) DNS admin: depool site eqiad [reason: junos upgrade, <a href="https://phabricator.wikimedia.org/T364092" class="extiw" title="phab:T364092">T364092</a>]</li> <li>15:36 sukhe@cumin1002: START - Cookbook sre.dns.admin DNS admin: depool site eqiad [reason: junos upgrade, <a href="https://phabricator.wikimedia.org/T364092" class="extiw" title="phab:T364092">T364092</a>]</li> <li>15:35 ladsgroup@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091248" class="extiw" title="gerrit:1091248">Revert "mmv.js: Store comingFromHashChange as a class property" (T379835)</a> (duration: 12m 10s)</li> <li>15:33 sukhe: reprepro -C main include bullseye-wikimedia trafficserver_9.2.6-1wm1_amd64.changes: <a href="https://phabricator.wikimedia.org/T379797" class="extiw" title="phab:T379797">T379797</a></li> <li>15:30 sukhe@cumin1002: START - Cookbook sre.dns.roll-restart-ntp rolling restart_daemons on A:dnsbox</li> <li>15:29 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl2002.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T379719" class="extiw" title="phab:T379719">T379719</a></li> <li>15:29 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl2002.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T379719" class="extiw" title="phab:T379719">T379719</a></li> <li>15:28 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-ctrl2002.codfw.wmnet</li> <li>15:28 jayme@cumin2002: START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-ctrl2002.codfw.wmnet</li> <li>15:27 ladsgroup@deploy2002: ladsgroup: Continuing with sync</li> <li>15:27 ladsgroup@deploy2002: ladsgroup: Backport for <a href="https://gerrit.wikimedia.org/r/1091248" class="extiw" title="gerrit:1091248">Revert "mmv.js: Store comingFromHashChange as a class property" (T379835)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:24 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>15:24 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>15:24 sukhe@cumin1002: END (PASS) - Cookbook sre.dns.roll-restart (exit_code=0) rolling restart_daemons on A:dnsbox and not A:magru and A:dnsbox</li> <li>15:23 ladsgroup@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091248" class="extiw" title="gerrit:1091248">Revert "mmv.js: Store comingFromHashChange as a class property" (T379835)</a></li> <li>15:16 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-search: apply</li> <li>15:15 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-search: apply</li> <li>15:07 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>15:07 sergi0: UTC afternoon deploys done</li> <li>15:06 sgimeno@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091231" class="extiw" title="gerrit:1091231">HomepageHooks: run metrics increment in deferred update (T379682)</a> (duration: 11m 15s)</li> <li>15:02 elukey@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>15:02 sgimeno@deploy2002: sgimeno: Continuing with sync</li> <li>14:59 sgimeno@deploy2002: sgimeno: Backport for <a href="https://gerrit.wikimedia.org/r/1091231" class="extiw" title="gerrit:1091231">HomepageHooks: run metrics increment in deferred update (T379682)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:55 sgimeno@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091231" class="extiw" title="gerrit:1091231">HomepageHooks: run metrics increment in deferred update (T379682)</a></li> <li>14:53 volans: uploaded spicerack_8.16.1 to apt.wikimedia.org bullseye-wikimedia</li> <li>14:50 sgimeno@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090830" class="extiw" title="gerrit:1090830">GrowthExperiments: set experiment config only in pilot wikis (T379681)</a> (duration: 13m 02s)</li> <li>14:45 sgimeno@deploy2002: sgimeno: Continuing with sync</li> <li>14:41 sgimeno@deploy2002: sgimeno: Backport for <a href="https://gerrit.wikimedia.org/r/1090830" class="extiw" title="gerrit:1090830">GrowthExperiments: set experiment config only in pilot wikis (T379681)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:37 sgimeno@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090830" class="extiw" title="gerrit:1090830">GrowthExperiments: set experiment config only in pilot wikis (T379681)</a></li> <li>14:33 sukhe@cumin1002: START - Cookbook sre.dns.roll-restart rolling restart_daemons on A:dnsbox and not A:magru and A:dnsbox</li> <li>14:30 sukhe@cumin1002: END (PASS) - Cookbook sre.dns.roll-restart (exit_code=0) rolling restart_daemons on A:dnsbox and A:magru and A:dnsbox</li> <li>14:27 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091227" class="extiw" title="gerrit:1091227">CX3 Build 0.2.0+20241114</a> (duration: 13m 23s)</li> <li>14:25 sukhe@cumin1002: START - Cookbook sre.dns.roll-restart rolling restart_daemons on A:dnsbox and A:magru and A:dnsbox</li> <li>14:22 kartik@deploy2002: kartik: Continuing with sync</li> <li>14:18 sukhe@cumin1002: END (PASS) - Cookbook sre.dns.roll-restart-reboot-wikimedia-dns (exit_code=0) rolling restart_daemons on A:wikidough and A:wikidough</li> <li>14:17 kartik@deploy2002: kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1091227" class="extiw" title="gerrit:1091227">CX3 Build 0.2.0+20241114</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:13 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091227" class="extiw" title="gerrit:1091227">CX3 Build 0.2.0+20241114</a></li> <li>14:05 sukhe@cumin1002: START - Cookbook sre.dns.roll-restart-reboot-wikimedia-dns rolling restart_daemons on A:wikidough and A:wikidough</li> <li>13:50 aqu@deploy2002: Finished deploy [airflow-dags/analytics@2220747]: Stage Refine parallelization improvment [airflow-dags@2220747d] (duration: 01m 08s)</li> <li>13:49 aqu@deploy2002: Started deploy [airflow-dags/analytics@2220747]: Stage Refine parallelization improvment [airflow-dags@2220747d]</li> <li>13:38 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti7004.magru.wmnet</li> <li>13:36 aqu@deploy2002: Finished deploy [airflow-dags/analytics_test@2220747]: Stage Refine parallelization improvment [airflow-dags@2220747d] (duration: 00m 15s)</li> <li>13:36 aqu@deploy2002: Started deploy [airflow-dags/analytics_test@2220747]: Stage Refine parallelization improvment [airflow-dags@2220747d]</li> <li>13:30 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti7004.magru.wmnet</li> <li>13:21 kcvelaga@deploy2002: Finished deploy [airflow-dags/analytics_product@c5ab766]: <a href="https://phabricator.wikimedia.org/T379546" class="extiw" title="phab:T379546">T379546</a> (duration: 00m 54s)</li> <li>13:21 kcvelaga@deploy2002: Started deploy [airflow-dags/analytics_product@c5ab766]: <a href="https://phabricator.wikimedia.org/T379546" class="extiw" title="phab:T379546">T379546</a></li> <li>13:19 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Fix search button height - oblivian@cumin1002"</li> <li>13:18 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Fix search button height - oblivian@cumin1002</li> <li>13:18 oblivian@cumin1002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Fix search button height - oblivian@cumin1002</li> <li>13:18 oblivian@cumin1002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Fix search button height - oblivian@cumin1002"</li> <li>13:05 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.reimage-stacked-control-plane (exit_code=0) Reimaging k8s control planes of cluster wikikube-codfw: containerd migration</li> <li>13:04 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-ctrl2003.codfw.wmnet with OS bookworm</li> <li>12:54 jmm@cumin2002: END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-eqiad</li> <li>12:53 jmm@cumin2002: START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-eqiad</li> <li>12:53 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti7004.magru.wmnet</li> <li>12:52 moritzm: installing apache2 security updates</li> <li>12:51 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti7004.magru.wmnet</li> <li>12:51 dreamyjazz@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090511" class="extiw" title="gerrit:1090511">Hide IP reveal tools on Special:AbuseLog and Special:GlobalBlockList (T379583)</a> (duration: 09m 08s)</li> <li>12:49 moritzm: failover ganeti master of magru02 to ganeti7002</li> <li>12:46 dreamyjazz@deploy2002: dreamyjazz: Continuing with sync</li> <li>12:45 dreamyjazz@deploy2002: dreamyjazz: Backport for <a href="https://gerrit.wikimedia.org/r/1090511" class="extiw" title="gerrit:1090511">Hide IP reveal tools on Special:AbuseLog and Special:GlobalBlockList (T379583)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>12:43 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti7002.magru.wmnet</li> <li>12:42 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl2003.codfw.wmnet with reason: host reimage</li> <li>12:41 dreamyjazz@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090511" class="extiw" title="gerrit:1090511">Hide IP reveal tools on Special:AbuseLog and Special:GlobalBlockList (T379583)</a></li> <li>12:38 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl2003.codfw.wmnet with reason: host reimage</li> <li>12:35 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti7002.magru.wmnet</li> <li>12:29 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti7002.magru.wmnet</li> <li>12:25 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti7002.magru.wmnet</li> <li>12:22 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl2003.codfw.wmnet with OS bookworm</li> <li>12:19 jmm@cumin2002: END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-codfw</li> <li>12:18 jmm@cumin2002: START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-codfw</li> <li>12:17 jayme@cumin2002: START - Cookbook sre.k8s.reimage-stacked-control-plane Reimaging k8s control planes of cluster wikikube-codfw: containerd migration</li> <li>12:10 jmm@cumin2002: END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling restart_daemons on A:ncredir</li> <li>12:00 jmm@cumin2002: START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling restart_daemons on A:ncredir</li> <li>11:57 moritzm: restarting postfix on inbound/outbound servers to pick up openssl updates</li> <li>11:17 moritzm: installing openssl security updates</li> <li>11:08 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.reimage-stacked-control-plane (exit_code=0) Reimaging k8s control planes of cluster wikikube-codfw: containerd migration</li> <li>11:08 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-ctrl2001.codfw.wmnet with OS bookworm</li> <li>10:47 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s_services/services/datahub: sync on production</li> <li>10:45 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl2001.codfw.wmnet with reason: host reimage</li> <li>10:44 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s_services/services/datahub: apply on production</li> <li>10:42 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl2001.codfw.wmnet with reason: host reimage</li> <li>10:16 moritzm: remove ganeti2017 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T376594" class="extiw" title="phab:T376594">T376594</a></li> <li>10:15 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2017.codfw.wmnet</li> <li>10:11 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl2001.codfw.wmnet with OS bookworm</li> <li>10:07 jnuche@deploy2002: Finished deploy [releng/jenkins-deploy@34b35a5] (releasing): (no justification provided) (duration: 00m 47s)</li> <li>10:06 jayme@cumin2002: START - Cookbook sre.k8s.reimage-stacked-control-plane Reimaging k8s control planes of cluster wikikube-codfw: containerd migration</li> <li>10:06 jnuche@deploy2002: Started deploy [releng/jenkins-deploy@34b35a5] (releasing): (no justification provided)</li> <li>10:03 jnuche@deploy2002: Finished deploy [releng/jenkins-deploy@34b35a5] (releasing): (no justification provided) (duration: 00m 21s)</li> <li>10:03 jnuche@deploy2002: Started deploy [releng/jenkins-deploy@34b35a5] (releasing): (no justification provided)</li> <li>09:43 kart_: Done: UTC morning backport window</li> <li>09:37 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090988" class="extiw" title="gerrit:1090988">Correction to virtual-globaljsonlinks mapping (T374746)</a> (duration: 10m 03s)</li> <li>09:37 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>09:36 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>09:35 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>09:34 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>09:32 kartik@deploy2002: bvibber, kartik: Continuing with sync</li> <li>09:31 kartik@deploy2002: bvibber, kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1090988" class="extiw" title="gerrit:1090988">Correction to virtual-globaljsonlinks mapping (T374746)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>09:27 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090988" class="extiw" title="gerrit:1090988">Correction to virtual-globaljsonlinks mapping (T374746)</a></li> <li>09:25 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091007" class="extiw" title="gerrit:1091007">CX3 Build 0.2.0+20241113 (T368718 T374567)</a> (duration: 29m 40s)</li> <li>09:21 kartik@deploy2002: kartik: Continuing with sync</li> <li>09:17 volans: installed spicerack v8.16.0 on cumin2002</li> <li>09:08 vgutierrez@cumin1002: END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on P{cp4044.ulsfo.wmnet,cp4052.ulsfo.wmnet} and A:cp</li> <li>09:04 vgutierrez@cumin1002: START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on P{cp4044.ulsfo.wmnet,cp4052.ulsfo.wmnet} and A:cp</li> <li>09:00 kartik@deploy2002: kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1091007" class="extiw" title="gerrit:1091007">CX3 Build 0.2.0+20241113 (T368718 T374567)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:56 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1091007" class="extiw" title="gerrit:1091007">CX3 Build 0.2.0+20241113 (T368718 T374567)</a></li> <li>08:55 vgutierrez: import haproxy 2.8.12 to thirtdparty/haproxy28 component for bullseye-wikimedia (apt.wm.o) - <a href="https://phabricator.wikimedia.org/T379891" class="extiw" title="phab:T379891">T379891</a></li> <li>08:54 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090937" class="extiw" title="gerrit:1090937">Allow Wikidata bureaucrats to remove admin rights (T379635)</a> (duration: 11m 49s)</li> <li>08:49 kartik@deploy2002: dreamrimmer, kartik: Continuing with sync</li> <li>08:47 kartik@deploy2002: dreamrimmer, kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1090937" class="extiw" title="gerrit:1090937">Allow Wikidata bureaucrats to remove admin rights (T379635)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:42 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090937" class="extiw" title="gerrit:1090937">Allow Wikidata bureaucrats to remove admin rights (T379635)</a></li> <li>08:38 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 26744</li> <li>08:37 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 26744</li> <li>08:35 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 141082</li> <li>08:35 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 141082</li> <li>08:34 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 9299</li> <li>08:33 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 9299</li> <li>08:33 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 140407</li> <li>08:33 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 140407</li> <li>08:28 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1084704" class="extiw" title="gerrit:1084704">Update stream registration and config for MinT for Readers (T378565)</a> (duration: 24m 50s)</li> <li>08:23 kartik@deploy2002: kcvelaga, kartik: Continuing with sync</li> <li>08:08 kartik@deploy2002: kcvelaga, kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1084704" class="extiw" title="gerrit:1084704">Update stream registration and config for MinT for Readers (T378565)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:03 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1084704" class="extiw" title="gerrit:1084704">Update stream registration and config for MinT for Readers (T378565)</a></li> <li>07:42 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2017.codfw.wmnet</li> <li>07:41 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2017.codfw.wmnet</li> <li>07:34 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2017.codfw.wmnet</li> <li>07:34 ayounsi@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>07:34 ayounsi@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Remove office link dns records - ayounsi@cumin1002"</li> <li>07:34 ayounsi@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Remove office link dns records - ayounsi@cumin1002"</li> <li>07:30 ayounsi@cumin1002: START - Cookbook sre.dns.netbox</li> <li>07:06 XioNoX: delete office interco IP/prefixes/vlan in ulsfo - <a href="https://phabricator.wikimedia.org/T379778" class="extiw" title="phab:T379778">T379778</a></li> <li>04:34 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>04:11 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>04:09 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>03:56 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>02:32 eileen: config revision changed from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7af5769b"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7af5769b">7af5769b</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="fbddc1f5"><a class="external text" href="https://gerrit.wikimedia.org/r/q/fbddc1f5">fbddc1f5</a></span></li> <li>02:29 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7b300007"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7b300007">7b300007</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="2ab8334a"><a class="external text" href="https://gerrit.wikimedia.org/r/q/2ab8334a">2ab8334a</a></span></li> <li>00:14 eileen: config revision changed from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="2b08b881"><a class="external text" href="https://gerrit.wikimedia.org/r/q/2b08b881">2b08b881</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7af5769b"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7af5769b">7af5769b</a></span></li> <li>00:13 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es1046.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>00:13 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>00:12 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="23e08fc2"><a class="external text" href="https://gerrit.wikimedia.org/r/q/23e08fc2">23e08fc2</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7b300007"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7b300007">7b300007</a></span></li> <li>00:05 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es1043.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>00:05 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es1042.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>00:05 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es1045.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>00:05 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host es1041.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-13" data-mw-thread-id="h-2024-11-13"><span data-mw-comment-start="" id="h-2024-11-13"></span>2024-11-13<span data-mw-comment-end="h-2024-11-13"></span></h2></div> <ul><li>23:45 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:43 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1042.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:43 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host es1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:43 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host es1042.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1046.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1045.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1043.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1042.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:42 jclark@cumin1002: START - Cookbook sre.hosts.provision for host es1041.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:41 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>23:41 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for es104 - jclark@cumin1002"</li> <li>23:41 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for es104 - jclark@cumin1002"</li> <li>23:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wdqs1027.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wdqs1026.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wdqs1025.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>23:37 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>23:20 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>23:04 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>23:04 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>23:04 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>22:59 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>22:58 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wdqs1025.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:58 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wdqs1026.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:58 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wdqs1027.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:57 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:55 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:33 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>22:33 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>22:30 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>22:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>22:25 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>22:21 jforrester@deploy2002: helmfile [eqiad] DONE helmfile.d/services/wikifunctions: apply</li> <li>22:20 jforrester@deploy2002: helmfile [eqiad] START helmfile.d/services/wikifunctions: apply</li> <li>22:20 jforrester@deploy2002: helmfile [codfw] DONE helmfile.d/services/wikifunctions: apply</li> <li>22:19 jforrester@deploy2002: helmfile [codfw] START helmfile.d/services/wikifunctions: apply</li> <li>22:18 jforrester@deploy2002: helmfile [staging] DONE helmfile.d/services/wikifunctions: apply</li> <li>22:17 jforrester@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>22:14 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>22:11 jforrester@deploy2002: helmfile [eqiad] DONE helmfile.d/services/wikifunctions: apply</li> <li>22:11 jforrester@deploy2002: helmfile [eqiad] START helmfile.d/services/wikifunctions: apply</li> <li>22:10 jforrester@deploy2002: helmfile [codfw] DONE helmfile.d/services/wikifunctions: apply</li> <li>22:10 jforrester@deploy2002: helmfile [codfw] START helmfile.d/services/wikifunctions: apply</li> <li>22:09 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>22:04 jforrester@deploy2002: helmfile [staging] DONE helmfile.d/services/wikifunctions: apply</li> <li>22:03 jforrester@deploy2002: helmfile [staging] START helmfile.d/services/wikifunctions: apply</li> <li>22:00 tchanders@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090965" class="extiw" title="gerrit:1090965">Revert "Disallow AbuseFilter protected variables use on non-temp-user wikis" (T379503)</a> (duration: 09m 03s)</li> <li>21:55 tchanders@deploy2002: tchanders: Continuing with sync</li> <li>21:55 tchanders@deploy2002: tchanders: Backport for <a href="https://gerrit.wikimedia.org/r/1090965" class="extiw" title="gerrit:1090965">Revert "Disallow AbuseFilter protected variables use on non-temp-user wikis" (T379503)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:51 tchanders@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090965" class="extiw" title="gerrit:1090965">Revert "Disallow AbuseFilter protected variables use on non-temp-user wikis" (T379503)</a></li> <li>21:48 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090953" class="extiw" title="gerrit:1090953">Enable autocreateaccount on testcommonswiki (T378216)</a> (duration: 12m 59s)</li> <li>21:44 cjming@deploy2002: aude, cjming: Continuing with sync</li> <li>21:40 cjming@deploy2002: aude, cjming: Backport for <a href="https://gerrit.wikimedia.org/r/1090953" class="extiw" title="gerrit:1090953">Enable autocreateaccount on testcommonswiki (T378216)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:36 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>21:36 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090953" class="extiw" title="gerrit:1090953">Enable autocreateaccount on testcommonswiki (T378216)</a></li> <li>21:34 cjming@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090928" class="extiw" title="gerrit:1090928">GlobalJsonLinksCachePurgeJob to actually invalidate caches (T374746)</a> (duration: 13m 27s)</li> <li>21:27 cjming@deploy2002: cjming, bvibber: Continuing with sync</li> <li>21:27 cjming@deploy2002: cjming, bvibber: Backport for <a href="https://gerrit.wikimedia.org/r/1090928" class="extiw" title="gerrit:1090928">GlobalJsonLinksCachePurgeJob to actually invalidate caches (T374746)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:21 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:21 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:21 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>21:20 cjming@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090928" class="extiw" title="gerrit:1090928">GlobalJsonLinksCachePurgeJob to actually invalidate caches (T374746)</a></li> <li>21:19 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>21:16 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>21:15 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>21:09 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:09 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:09 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:09 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:07 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:07 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>21:07 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host thanos-be2005</li> <li>21:07 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host thanos-be2005</li> <li>21:05 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>21:05 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>21:01 aqu@deploy2002: Finished deploy [airflow-dags/analytics@3487da3]: Stage Refine [airflow-dags@3487da3a] (duration: 01m 22s)</li> <li>21:00 aqu@deploy2002: Started deploy [airflow-dags/analytics@3487da3]: Stage Refine [airflow-dags@3487da3a]</li> <li>20:56 aqu@deploy2002: Finished deploy [airflow-dags/analytics@3fc12d6]: Stage Refine [airflow-dags@3fc12d60] (duration: 01m 14s)</li> <li>20:56 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>20:56 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>20:55 aqu@deploy2002: Started deploy [airflow-dags/analytics@3fc12d6]: Stage Refine [airflow-dags@3fc12d60]</li> <li>20:49 cdanis@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>20:49 cdanis@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>20:48 swfrench-wmf: deployed changeprop to clear no-op chart version diffs from CR <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1089313"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1089313/">1089313</a></span></li> <li>20:47 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/changeprop: apply</li> <li>20:47 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/changeprop: apply</li> <li>20:46 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>20:39 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>20:37 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop: apply</li> <li>20:37 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop: apply</li> <li>20:36 cdanis@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>20:36 cdanis@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>20:35 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop: apply</li> <li>20:34 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/changeprop: apply</li> <li>20:34 aqu@deploy2002: Finished deploy [airflow-dags/analytics_test@3fc12d6]: Stage Refine [airflow-dags@3fc12d60] (duration: 00m 15s)</li> <li>20:34 aqu@deploy2002: Started deploy [airflow-dags/analytics_test@3fc12d6]: Stage Refine [airflow-dags@3fc12d60]</li> <li>20:31 cdanis@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>20:31 cdanis@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>20:28 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>20:28 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>20:16 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>20:14 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>20:02 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>20:02 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>19:59 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>19:59 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>19:59 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host thanos-be2005</li> <li>19:59 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host thanos-be2005</li> <li>19:58 cdanis@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>19:58 cdanis@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>19:58 brennen@deploy2002: Finished scap sync-world: testwikis to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a> (duration: 31m 07s)</li> <li>19:57 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>19:55 cdanis@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>19:55 cdanis@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>19:52 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host thanos-be2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>19:51 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>19:51 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding thanos-be2005 to codfw - jhancock@cumin2002"</li> <li>19:51 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding thanos-be2005 to codfw - jhancock@cumin2002"</li> <li>19:47 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>19:47 cdanis@deploy2002: helmfile [aux-k8s-eqiad] DONE helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>19:46 cdanis@deploy2002: helmfile [aux-k8s-eqiad] START helmfile.d/aus-k8s-eqiad-services/jaeger: apply</li> <li>19:44 aokoth@cumin1002: END (PASS) - Cookbook sre.gitlab.upgrade (exit_code=0) on GitLab host gitlab1003.wikimedia.org with reason: Security Update</li> <li>19:37 aokoth@cumin1002: START - Cookbook sre.gitlab.upgrade on GitLab host gitlab1003.wikimedia.org with reason: Security Update</li> <li>19:36 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>19:35 aokoth@cumin1002: END (PASS) - Cookbook sre.gitlab.upgrade (exit_code=0) on GitLab host gitlab1004.wikimedia.org with reason: Security Update</li> <li>19:27 brennen@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>19:26 brennen@deploy2002: rebuilt and synchronized wikiversions files: group1 to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>19:21 aokoth@cumin1002: START - Cookbook sre.gitlab.upgrade on GitLab host gitlab1004.wikimedia.org with reason: Security Update</li> <li>19:13 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host thanos-be1005.eqiad.wmnet with OS bullseye</li> <li>19:11 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:10 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:10 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:10 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:09 brennen: 1.44.0-wmf.3 train status (<a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a>): no current blockers, rolling to group1.</li> <li>19:08 bking@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/hdfs-synchronizer: apply</li> <li>19:03 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:03 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:02 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:02 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:01 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:01 jclark@cumin1002: START - Cookbook sre.hosts.provision for host thanos-be1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>19:00 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>19:00 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for thanos-be1005 - jclark@cumin1002"</li> <li>19:00 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for thanos-be1005 - jclark@cumin1002"</li> <li>18:58 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/hdfs-synchronizer: apply</li> <li>18:56 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:50 swfrench@deploy2002: Finished scap sync-world: Deployment to switch mwdebug-next to publish-81 - <a href="https://phabricator.wikimedia.org/T372604" class="extiw" title="phab:T372604">T372604</a> (duration: 01m 53s)</li> <li>18:48 swfrench@deploy2002: Started scap sync-world: Deployment to switch mwdebug-next to publish-81 - <a href="https://phabricator.wikimedia.org/T372604" class="extiw" title="phab:T372604">T372604</a></li> <li>18:36 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/mw-debug: apply</li> <li>18:33 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/mw-debug: apply</li> <li>18:32 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply</li> <li>18:30 cdanis@deploy2002: Finished deploy [docker-pkg/deploy@3499887]: I really hope this works this time (duration: 00m 34s)</li> <li>18:29 cdanis@deploy2002: Started deploy [docker-pkg/deploy@3499887]: I really hope this works this time</li> <li>18:29 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/mw-debug: apply</li> <li>18:26 cdanis@deploy2002: Finished deploy [docker-pkg/deploy@9d71ac3]: (no justification provided) (duration: 00m 18s)</li> <li>18:26 cdanis@deploy2002: Started deploy [docker-pkg/deploy@9d71ac3]: (no justification provided)</li> <li>18:22 cdanis@deploy2002: Finished deploy [docker-pkg/deploy@9d71ac3]: (no justification provided) (duration: 00m 40s)</li> <li>18:21 cdanis@deploy2002: Started deploy [docker-pkg/deploy@9d71ac3]: (no justification provided)</li> <li>18:21 cdanis@deploy2002: Finished deploy [docker-pkg/deploy@9d71ac3]: deploy 4.0.2 for realsies (duration: 02m 41s)</li> <li>18:18 cdanis@deploy2002: Started deploy [docker-pkg/deploy@9d71ac3]: deploy 4.0.2 for realsies</li> <li>18:13 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>18:13 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 3:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>18:11 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:54 urbanecm: mwmaint2002: foreachwikiindblist growthexperiments extensions/GrowthExperiments/maintenance/fixLinkRecommendationData.php --search-index --verbose --random # <a href="https://phabricator.wikimedia.org/T379057" class="extiw" title="phab:T379057">T379057</a></li> <li>17:49 cdanis@deploy2002: Finished deploy [docker-pkg/deploy@38eb04d]: ship upstream_version helper (duration: 00m 32s)</li> <li>17:49 cdanis@deploy2002: Started deploy [docker-pkg/deploy@38eb04d]: ship upstream_version helper</li> <li>17:49 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>17:47 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:46 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>17:45 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:40 jayme@cumin1002: conftool action : set/pooled=yes; selector: name=wikikube-ctrl2002.codfw.wmnet</li> <li>17:39 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for wikikube-ctrl2002.codfw.wmnet</li> <li>17:39 jayme@cumin2002: START - Cookbook sre.hosts.remove-downtime for wikikube-ctrl2002.codfw.wmnet</li> <li>17:38 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-ctrl2002.codfw.wmnet with OS bookworm</li> <li>17:37 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:35 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:33 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:32 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:23 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker[2128-2135].codfw.wmnet</li> <li>17:23 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker[2128-2135].codfw.wmnet</li> <li>17:20 claime: homer 'lsw1-d2-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a>'</li> <li>17:18 claime: homer 'lsw1-c2-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a>'</li> <li>17:18 claime: homer 'lsw1-d4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a>'</li> <li>17:17 claime: homer 'lsw1-c4-codfw*' commit '<a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a>'</li> <li>17:15 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:14 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl2002.codfw.wmnet with reason: host reimage</li> <li>17:11 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl2002.codfw.wmnet with reason: host reimage</li> <li>17:03 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:02 claime: homer 'cr*codfw*' commit <a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a></li> <li>17:01 claime: homer 'lsw1-b4-codfw*' commit <a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a></li> <li>17:01 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>16:58 claime: homer 'lsw1-b2-codfw*' commit <a href="https://phabricator.wikimedia.org/T377008" class="extiw" title="phab:T377008">T377008</a></li> <li>16:53 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/growthbook: apply</li> <li>16:53 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.move-vlan (exit_code=0) for host wikikube-ctrl2002</li> <li>16:53 jayme@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host wikikube-ctrl2002</li> <li>16:53 jayme@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-ctrl2002</li> <li>16:53 jayme@cumin2002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) wikikube-ctrl2002.codfw.wmnet 76.32.192.10.in-addr.arpa 6.7.0.0.2.3.0.0.2.9.1.0.0.1.0.0.3.0.1.0.0.6.8.0.0.0.0.0.0.2.6.2.ip6.arpa on all recursors</li> <li>16:53 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/growthbook: apply</li> <li>16:53 jayme@cumin2002: START - Cookbook sre.dns.wipe-cache wikikube-ctrl2002.codfw.wmnet 76.32.192.10.in-addr.arpa 6.7.0.0.2.3.0.0.2.9.1.0.0.1.0.0.3.0.1.0.0.6.8.0.0.0.0.0.0.2.6.2.ip6.arpa on all recursors</li> <li>16:53 jayme@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:53 jayme@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Update records for host wikikube-ctrl2002 - jayme@cumin2002"</li> <li>16:53 jayme@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Update records for host wikikube-ctrl2002 - jayme@cumin2002"</li> <li>16:50 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2135.codfw.wmnet with OS bookworm</li> <li>16:49 jayme@cumin2002: START - Cookbook sre.dns.netbox</li> <li>16:48 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2134.codfw.wmnet with OS bookworm</li> <li>16:47 jayme@cumin2002: START - Cookbook sre.hosts.move-vlan for host wikikube-ctrl2002</li> <li>16:47 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/growthbook: apply</li> <li>16:47 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl2002.codfw.wmnet with OS bookworm</li> <li>16:47 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/growthbook: apply</li> <li>16:41 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on wikikube-ctrl2002.codfw.wmnet with reason: reimage</li> <li>16:40 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on wikikube-ctrl2002.codfw.wmnet with reason: reimage</li> <li>16:37 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti7003.magru.wmnet</li> <li>16:31 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2135.codfw.wmnet with reason: host reimage</li> <li>16:31 jayme@cumin2002: conftool action : set/pooled=inactive; selector: name=wikikube-ctrl2002.codfw.wmnet</li> <li>16:30 elukey: reload nginx on registry* to pick up logging changes (log of X-Client-IP from the CDN)</li> <li>16:30 XioNoX: shutdown old office link interface - <a href="https://phabricator.wikimedia.org/T379778" class="extiw" title="phab:T379778">T379778</a></li> <li>16:29 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2133.codfw.wmnet with OS bookworm</li> <li>16:29 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2134.codfw.wmnet with reason: host reimage</li> <li>16:29 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti7003.magru.wmnet</li> <li>16:26 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2135.codfw.wmnet with reason: host reimage</li> <li>16:25 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2134.codfw.wmnet with reason: host reimage</li> <li>16:24 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2132.codfw.wmnet with OS bookworm</li> <li>16:15 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti7003.magru.wmnet</li> <li>16:14 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti7003.magru.wmnet</li> <li>16:08 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2133.codfw.wmnet with reason: host reimage</li> <li>16:08 sukhe: running agent on A:ulsfo and A:lvs</li> <li>16:07 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2135.codfw.wmnet with OS bookworm</li> <li>16:06 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2134.codfw.wmnet with OS bookworm</li> <li>16:05 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2132.codfw.wmnet with reason: host reimage</li> <li>16:04 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2133.codfw.wmnet with reason: host reimage</li> <li>16:02 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2132.codfw.wmnet with reason: host reimage</li> <li>15:56 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2131.codfw.wmnet with OS bookworm</li> <li>15:53 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2130.codfw.wmnet with OS bookworm</li> <li>15:47 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>15:47 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>15:45 bking@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/hdfs-synchronizer: apply</li> <li>15:45 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2133.codfw.wmnet with OS bookworm</li> <li>15:42 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2132.codfw.wmnet with OS bookworm</li> <li>15:37 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2129.codfw.wmnet with OS bookworm</li> <li>15:37 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2131.codfw.wmnet with reason: host reimage</li> <li>15:36 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>15:35 moritzm: failover ganeti master of magru01 to ganeti7001</li> <li>15:34 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2130.codfw.wmnet with reason: host reimage</li> <li>15:33 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2131.codfw.wmnet with reason: host reimage</li> <li>15:33 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>15:33 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>15:30 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>15:30 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>15:30 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update dns records for IPs moving from old to new fundraising firewalls - cmooney@cumin1002"</li> <li>15:30 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update dns records for IPs moving from old to new fundraising firewalls - cmooney@cumin1002"</li> <li>15:30 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2130.codfw.wmnet with reason: host reimage</li> <li>15:26 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>15:23 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti7001.magru.wmnet</li> <li>15:18 moritzm: installing apache2 security updates</li> <li>15:18 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2129.codfw.wmnet with reason: host reimage</li> <li>15:15 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2131.codfw.wmnet with OS bookworm</li> <li>15:15 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2129.codfw.wmnet with reason: host reimage</li> <li>15:15 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti7001.magru.wmnet</li> <li>15:14 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>15:12 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2130.codfw.wmnet with OS bookworm</li> <li>14:59 volans: uploaded spicerack_8.16.0 to apt.wikimedia.org bullseye-wikimedia</li> <li>14:57 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2129.codfw.wmnet with OS bookworm</li> <li>14:56 aqu@deploy2002: Finished deploy [airflow-dags/analytics_test@2eb8320]: Stage Refine [airflow-dags@2eb8320d] (duration: 00m 14s)</li> <li>14:55 aqu@deploy2002: Started deploy [airflow-dags/analytics_test@2eb8320]: Stage Refine [airflow-dags@2eb8320d]</li> <li>14:55 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2128.codfw.wmnet with reason: host reimage</li> <li>14:51 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2128.codfw.wmnet with reason: host reimage</li> <li>14:51 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti7001.magru.wmnet</li> <li>14:50 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti7001.magru.wmnet</li> <li>14:37 moritzm: installing openssl security updates</li> <li>14:36 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2131.codfw.wmnet with OS bookworm</li> <li>14:36 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2130.codfw.wmnet with OS bookworm</li> <li>14:35 Lucas_WMDE: UTC afternoon backport+config window done</li> <li>14:33 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>14:32 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090526" class="extiw" title="gerrit:1090526">TimedMediahandler: reenable shellbox-video for commons (T356241)</a> (duration: 07m 28s)</li> <li>14:30 btullis@cumin1002: END (PASS) - Cookbook sre.kafka.roll-restart-reboot-brokers (exit_code=0) rolling restart_daemons on A:kafka-jumbo-eqiad</li> <li>14:27 lucaswerkmeister-wmde@deploy2002: hnowlan, lucaswerkmeister-wmde: Continuing with sync</li> <li>14:27 lucaswerkmeister-wmde@deploy2002: hnowlan, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1090526" class="extiw" title="gerrit:1090526">TimedMediahandler: reenable shellbox-video for commons (T356241)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:26 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-research: apply</li> <li>14:25 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-research: apply</li> <li>14:24 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090526" class="extiw" title="gerrit:1090526">TimedMediahandler: reenable shellbox-video for commons (T356241)</a></li> <li>14:21 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-research: apply</li> <li>14:21 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-research: apply</li> <li>14:15 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>14:14 tchanders@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090515" class="extiw" title="gerrit:1090515">Disallow AbuseFilter protected variables use on non-temp-user wikis (T379503)</a> (duration: 11m 28s)</li> <li>14:12 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-research: apply</li> <li>14:10 tchanders@deploy2002: tchanders: Continuing with sync</li> <li>14:09 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-research: apply</li> <li>14:07 akosiaris@deploy2002: helmfile [codfw] DONE helmfile.d/services/ipoid: apply</li> <li>14:07 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1052.eqiad.wmnet to cluster eqiad and group D</li> <li>14:07 akosiaris@deploy2002: helmfile [codfw] START helmfile.d/services/ipoid: apply</li> <li>14:06 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1052.eqiad.wmnet to cluster eqiad and group D</li> <li>14:06 tchanders@deploy2002: tchanders: Backport for <a href="https://gerrit.wikimedia.org/r/1090515" class="extiw" title="gerrit:1090515">Disallow AbuseFilter protected variables use on non-temp-user wikis (T379503)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:03 tchanders@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090515" class="extiw" title="gerrit:1090515">Disallow AbuseFilter protected variables use on non-temp-user wikis (T379503)</a></li> <li>14:03 akosiaris@deploy2002: helmfile [staging] DONE helmfile.d/services/ipoid: apply</li> <li>14:02 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/ipoid: apply</li> <li>14:01 akosiaris@deploy2002: helmfile [eqiad] DONE helmfile.d/services/ipoid: apply</li> <li>14:01 akosiaris@deploy2002: helmfile [eqiad] START helmfile.d/services/ipoid: apply</li> <li>14:00 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>13:59 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>13:32 btullis@cumin1002: START - Cookbook sre.kafka.roll-restart-reboot-brokers rolling restart_daemons on A:kafka-jumbo-eqiad</li> <li>13:21 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>13:20 hnowlan@deploy1003: helmfile [codfw] DONE helmfile.d/services/thumbor: apply</li> <li>13:18 moritzm: installing python-cryptography security updates</li> <li>13:18 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>13:18 btullis@cumin1002: END (PASS) - Cookbook sre.hadoop.roll-restart-masters (exit_code=0) restart masters for Hadoop test cluster: Restart of jvm daemons.</li> <li>13:17 hnowlan@deploy1003: helmfile [codfw] START helmfile.d/services/thumbor: apply</li> <li>13:14 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>13:13 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>13:12 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>13:11 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>13:09 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>13:08 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>13:08 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>13:07 hnowlan@deploy1003: helmfile [eqiad] DONE helmfile.d/services/thumbor: apply</li> <li>13:06 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>13:06 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>13:05 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:05 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:03 hnowlan@deploy1003: helmfile [eqiad] START helmfile.d/services/thumbor: apply</li> <li>12:59 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2129.codfw.wmnet with OS bookworm</li> <li>12:56 hnowlan@deploy1003: helmfile [staging] DONE helmfile.d/services/thumbor: apply</li> <li>12:56 hnowlan@deploy1003: helmfile [staging] START helmfile.d/services/thumbor: apply</li> <li>12:55 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>12:54 cgoubert@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>12:45 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>12:45 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1022 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71030">https://phabricator.wikimedia.org/P71030</a> and previous config saved to /var/cache/conftool/dbconfig/20241113-124504-ladsgroup.json</li> <li>12:44 cgoubert@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>12:33 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1051.eqiad.wmnet to cluster eqiad and group D</li> <li>12:32 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2131.codfw.wmnet with OS bookworm</li> <li>12:32 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1051.eqiad.wmnet to cluster eqiad and group D</li> <li>12:31 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>12:31 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2130.codfw.wmnet with OS bookworm</li> <li>12:30 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>12:29 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1022', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71029">https://phabricator.wikimedia.org/P71029</a> and previous config saved to /var/cache/conftool/dbconfig/20241113-122957-ladsgroup.json</li> <li>12:29 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2129.codfw.wmnet with OS bookworm</li> <li>12:29 fabfur@cumin1002: conftool action : set/pooled=yes; selector: name=cp5017.eqsin.wmnet</li> <li>12:28 cgoubert@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>12:28 btullis@cumin1002: END (PASS) - Cookbook sre.druid.roll-restart-workers (exit_code=0) for Druid test cluster: Roll restart of Druid jvm daemons.</li> <li>12:18 btullis@cumin1002: START - Cookbook sre.druid.roll-restart-workers for Druid test cluster: Roll restart of Druid jvm daemons.</li> <li>12:15 mvolz@deploy2002: helmfile [eqiad] DONE helmfile.d/services/zotero: apply</li> <li>12:15 mvolz@deploy2002: helmfile [eqiad] START helmfile.d/services/zotero: apply</li> <li>12:14 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1022', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71028">https://phabricator.wikimedia.org/P71028</a> and previous config saved to /var/cache/conftool/dbconfig/20241113-121450-ladsgroup.json</li> <li>12:14 mvolz@deploy2002: helmfile [codfw] DONE helmfile.d/services/zotero: apply</li> <li>12:14 mvolz@deploy2002: helmfile [codfw] START helmfile.d/services/zotero: apply</li> <li>12:13 mvolz@deploy2002: helmfile [staging] DONE helmfile.d/services/zotero: apply</li> <li>12:13 mvolz@deploy2002: helmfile [staging] START helmfile.d/services/zotero: apply</li> <li>12:11 mvolz@deploy2002: helmfile [staging] DONE helmfile.d/services/zotero: apply</li> <li>12:11 mvolz@deploy2002: helmfile [staging] START helmfile.d/services/zotero: apply</li> <li>12:06 mvolz@deploy2002: helmfile [eqiad] DONE helmfile.d/services/citoid: apply</li> <li>12:06 mvolz@deploy2002: helmfile [eqiad] START helmfile.d/services/citoid: apply</li> <li>12:06 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1052.eqiad.wmnet</li> <li>12:03 mvolz@deploy2002: helmfile [codfw] DONE helmfile.d/services/citoid: apply</li> <li>12:03 mvolz@deploy2002: helmfile [codfw] START helmfile.d/services/citoid: apply</li> <li>12:02 mvolz@deploy2002: helmfile [staging] DONE helmfile.d/services/citoid: apply</li> <li>12:01 mvolz@deploy2002: helmfile [staging] START helmfile.d/services/citoid: apply</li> <li>11:59 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1022 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71027">https://phabricator.wikimedia.org/P71027</a> and previous config saved to /var/cache/conftool/dbconfig/20241113-115943-ladsgroup.json</li> <li>11:57 jiji@deploy2002: helmfile [codfw] DONE helmfile.d/services/ipoid: apply</li> <li>11:57 jiji@deploy2002: helmfile [codfw] START helmfile.d/services/ipoid: apply</li> <li>11:57 jiji@deploy2002: helmfile [eqiad] DONE helmfile.d/services/ipoid: apply</li> <li>11:57 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1052.eqiad.wmnet</li> <li>11:57 jiji@deploy2002: helmfile [eqiad] START helmfile.d/services/ipoid: apply</li> <li>11:56 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1051.eqiad.wmnet</li> <li>11:55 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1052</li> <li>11:54 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1052</li> <li>11:52 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>11:51 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-wmde: apply</li> <li>11:51 hnowlan@deploy1003: helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>11:50 hnowlan@deploy1003: helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>11:49 hnowlan@deploy1003: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>11:49 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1022 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71026">https://phabricator.wikimedia.org/P71026</a> and previous config saved to /var/cache/conftool/dbconfig/20241113-114913-ladsgroup.json</li> <li>11:49 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1051.eqiad.wmnet</li> <li>11:49 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1022.eqiad.wmnet with reason: Maintenance</li> <li>11:48 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1022.eqiad.wmnet with reason: Maintenance</li> <li>11:48 hnowlan@deploy1003: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>11:47 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1051</li> <li>11:46 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>11:46 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1051</li> <li>11:45 stevemunene@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>11:41 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.reimage-stacked-control-plane (exit_code=0) Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>11:41 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-ctrl1003.eqiad.wmnet with OS bookworm</li> <li>11:34 hnowlan@deploy1003: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>11:34 hnowlan@deploy1003: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>11:26 cgoubert@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on wikikube-worker1256.eqiad.wmnet with reason: Degraded RAID</li> <li>11:26 cgoubert@cumin1002: START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on wikikube-worker1256.eqiad.wmnet with reason: Degraded RAID</li> <li>11:25 cgoubert@cumin1002: END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker1256.eqiad.wmnet</li> <li>11:25 cgoubert@cumin1002: START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker1256.eqiad.wmnet</li> <li>11:19 btullis@cumin1002: END (PASS) - Cookbook sre.druid.roll-restart-workers (exit_code=0) for Druid test cluster: Roll restart of Druid jvm daemons.</li> <li>11:18 btullis@cumin1002: START - Cookbook sre.hadoop.roll-restart-masters restart masters for Hadoop test cluster: Restart of jvm daemons.</li> <li>11:17 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl1003.eqiad.wmnet with reason: host reimage</li> <li>11:14 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl1003.eqiad.wmnet with reason: host reimage</li> <li>11:10 btullis@cumin1002: START - Cookbook sre.druid.roll-restart-workers for Druid test cluster: Roll restart of Druid jvm daemons.</li> <li>11:09 btullis@cumin1002: END (PASS) - Cookbook sre.druid.roll-restart-workers (exit_code=0) for Druid public cluster: Roll restart of Druid jvm daemons.</li> <li>10:42 ladsgroup@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090809" class="extiw" title="gerrit:1090809">Set the ratio of the new ParserCache keys to 100 for prod (T373037)</a> (duration: 07m 32s)</li> <li>10:37 ladsgroup@deploy2002: ladsgroup: Continuing with sync</li> <li>10:36 ladsgroup@deploy2002: ladsgroup: Backport for <a href="https://gerrit.wikimedia.org/r/1090809" class="extiw" title="gerrit:1090809">Set the ratio of the new ParserCache keys to 100 for prod (T373037)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>10:35 fabfur@cumin1002: conftool action : set/pooled=yes; selector: name=cp4037.ulsfo.wmnet</li> <li>10:34 ladsgroup@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090809" class="extiw" title="gerrit:1090809">Set the ratio of the new ParserCache keys to 100 for prod (T373037)</a></li> <li>10:32 btullis@cumin1002: END (PASS) - Cookbook sre.hadoop.roll-restart-workers (exit_code=0) restart workers for Hadoop test cluster: Roll restart of jvm daemons for openjdk upgrade.</li> <li>10:27 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1003.eqiad.wmnet with OS bookworm</li> <li>10:26 ladsgroup@deploy2002: ladsgroup: Continuing with sync</li> <li>10:26 jayme@cumin2002: START - Cookbook sre.k8s.reimage-stacked-control-plane Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>10:24 jayme@cumin2002: END (PASS) - Cookbook sre.k8s.reimage-stacked-control-plane (exit_code=0) Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>10:24 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-ctrl1002.eqiad.wmnet with OS bookworm</li> <li>10:21 fabfur@cumin1002: conftool action : set/pooled=no; selector: name=cp4037.ulsfo.wmnet</li> <li>10:20 btullis@cumin1002: START - Cookbook sre.hadoop.roll-restart-workers restart workers for Hadoop test cluster: Roll restart of jvm daemons for openjdk upgrade.</li> <li>10:20 ladsgroup@deploy2002: ladsgroup: Backport for <a href="https://gerrit.wikimedia.org/r/1090809" class="extiw" title="gerrit:1090809">Set the ratio of the new ParserCache keys to 100 for prod (T373037)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>10:18 btullis@cumin1002: START - Cookbook sre.druid.roll-restart-workers for Druid public cluster: Roll restart of Druid jvm daemons.</li> <li>10:17 ladsgroup@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090809" class="extiw" title="gerrit:1090809">Set the ratio of the new ParserCache keys to 100 for prod (T373037)</a></li> <li>10:09 elukey: disallow calls to /v2/_catalog from the outside internet on Docker Registry hosts - <a href="https://phabricator.wikimedia.org/T378618" class="extiw" title="phab:T378618">T378618</a></li> <li>10:04 claime: Manual restart of dump_cloud_ip_ranges.service on 'A:puppetserver or A:puppetmaster'</li> <li>10:01 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl1002.eqiad.wmnet with reason: host reimage</li> <li>10:01 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>10:00 elukey@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>10:00 elukey@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>09:55 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl1002.eqiad.wmnet with reason: host reimage</li> <li>09:41 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2088.codfw.wmnet with reason: host reimage</li> <li>09:38 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2088.codfw.wmnet with reason: host reimage</li> <li>09:25 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>09:20 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1002.eqiad.wmnet with OS bookworm</li> <li>09:20 jayme@cumin2002: START - Cookbook sre.k8s.reimage-stacked-control-plane Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>09:11 elukey@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>09:01 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>08:54 kart_: Updated recommedation-api to 2024-11-08-142328-production and fix wikidata host header (<a href="https://phabricator.wikimedia.org/T379592" class="extiw" title="phab:T379592">T379592</a>)</li> <li>08:49 kartik@deploy2002: helmfile [ml-serve-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>08:49 elukey@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>08:46 kartik@deploy2002: helmfile [ml-serve-eqiad] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>08:33 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2088.codfw.wmnet with reason: host reimage</li> <li>08:27 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2088.codfw.wmnet with reason: host reimage</li> <li>08:14 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>08:13 ladsgroup@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090493" class="extiw" title="gerrit:1090493">Revert "cswiki: Add celebration logo"</a> (duration: 09m 18s)</li> <li>08:08 ladsgroup@deploy2002: ladsgroup, hamishz: Continuing with sync</li> <li>08:07 ladsgroup@deploy2002: ladsgroup, hamishz: Backport for <a href="https://gerrit.wikimedia.org/r/1090493" class="extiw" title="gerrit:1090493">Revert "cswiki: Add celebration logo"</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:06 kartik@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>08:04 ladsgroup@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090493" class="extiw" title="gerrit:1090493">Revert "cswiki: Add celebration logo"</a></li> <li>07:47 Amir1: running extensions/Echo/maintenance/removeOrphanedEvents.php --force on all wikis (<a href="https://phabricator.wikimedia.org/T308084" class="extiw" title="phab:T308084">T308084</a>)</li> <li>05:17 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="ad008134"><a class="external text" href="https://gerrit.wikimedia.org/r/q/ad008134">ad008134</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="23e08fc2"><a class="external text" href="https://gerrit.wikimedia.org/r/q/23e08fc2">23e08fc2</a></span></li> <li>02:56 tchin@deploy2002: Finished deploy [airflow-dags/analytics@58d7b82]: (no justification provided) (duration: 00m 10s)</li> <li>02:56 tchin@deploy2002: Started deploy [airflow-dags/analytics@58d7b82]: (no justification provided)</li> <li>02:55 tchin@deploy2002: deploy aborted: failedpythonlol (duration: 00m 05s)</li> <li>02:55 tchin@deploy2002: Started deploy [airflow-dags/analytics@58d7b82]: failedpythonlol</li> <li>00:54 tchin@deploy2002: Started deploy [airflow-dags/analytics@58d7b82]: (no justification provided)</li> <li>00:35 ejegg: payments-wiki upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="7d24a942"><a class="external text" href="https://gerrit.wikimedia.org/r/q/7d24a942">7d24a942</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="459f259b"><a class="external text" href="https://gerrit.wikimedia.org/r/q/459f259b">459f259b</a></span></li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-12" data-mw-thread-id="h-2024-11-12"><span data-mw-comment-start="" id="h-2024-11-12"></span>2024-11-12<span data-mw-comment-end="h-2024-11-12"></span></h2></div> <ul><li>23:28 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>23:11 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>23:08 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:35 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>22:11 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>21:55 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>21:55 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>21:28 ebysans@deploy2002: Finished deploy [airflow-dags/analytics@58d7b82]: (no justification provided) (duration: 03m 50s)</li> <li>21:27 SandraEbele_: deploying airflow as part of weekly deployment train</li> <li>21:27 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088770" class="extiw" title="gerrit:1088770">Fix warning about missing central account for temp users (T378289)</a>, <a href="https://gerrit.wikimedia.org/r/1088771" class="extiw" title="gerrit:1088771">Check session provider when autocreating (T378289)</a> (duration: 16m 11s)</li> <li>21:25 ebysans@deploy2002: Started deploy [airflow-dags/analytics@58d7b82]: (no justification provided)</li> <li>21:23 SandraEbele_: Deployed refinery using scap, then deployed onto hdfs</li> <li>21:22 urbanecm@deploy2002: urbanecm, tgr: Continuing with sync</li> <li>21:22 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>21:13 urbanecm@deploy2002: urbanecm, tgr: Backport for <a href="https://gerrit.wikimedia.org/r/1088770" class="extiw" title="gerrit:1088770">Fix warning about missing central account for temp users (T378289)</a>, <a href="https://gerrit.wikimedia.org/r/1088771" class="extiw" title="gerrit:1088771">Check session provider when autocreating (T378289)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:11 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088770" class="extiw" title="gerrit:1088770">Fix warning about missing central account for temp users (T378289)</a>, <a href="https://gerrit.wikimedia.org/r/1088771" class="extiw" title="gerrit:1088771">Check session provider when autocreating (T378289)</a></li> <li>21:09 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090550" class="extiw" title="gerrit:1090550">Revert^2 "[CirrusSearch] testwiki: enable offloading weighted tags via EventBus" (T378983)</a> (duration: 07m 18s)</li> <li>21:04 ebysans@deploy2002: Finished deploy [analytics/refinery@113ea5a] (hadoop-test): Regular analytics weekly train TEST [analytics/refinery@113ea5ac] (duration: 04m 09s)</li> <li>21:02 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090550" class="extiw" title="gerrit:1090550">Revert^2 "[CirrusSearch] testwiki: enable offloading weighted tags via EventBus" (T378983)</a></li> <li>20:59 ebysans@deploy2002: Started deploy [analytics/refinery@113ea5a] (hadoop-test): Regular analytics weekly train TEST [analytics/refinery@113ea5ac]</li> <li>20:59 ebysans@deploy2002: Finished deploy [analytics/refinery@113ea5a] (thin): Regular analytics weekly train THIN [analytics/refinery@113ea5ac] (duration: 04m 54s)</li> <li>20:54 ebysans@deploy2002: Started deploy [analytics/refinery@113ea5a] (thin): Regular analytics weekly train THIN [analytics/refinery@113ea5ac]</li> <li>20:53 ebysans@deploy2002: Finished deploy [analytics/refinery@113ea5a]: Regular analytics weekly train [analytics/refinery@113ea5ac] (duration: 07m 37s)</li> <li>20:49 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'experimental' for release 'main' .</li> <li>20:46 ebysans@deploy2002: Started deploy [analytics/refinery@113ea5a]: Regular analytics weekly train [analytics/refinery@113ea5ac]</li> <li>19:42 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for wikikube-ctrl1001.eqiad.wmnet</li> <li>19:42 jayme@cumin2002: START - Cookbook sre.hosts.remove-downtime for wikikube-ctrl1001.eqiad.wmnet</li> <li>19:42 jayme@cumin2002: conftool action : set/pooled=yes; selector: name=wikikube-ctrl1001.*</li> <li>19:40 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>19:16 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-ctrl1001.eqiad.wmnet with reason: host reimage</li> <li>19:14 brennen@deploy2002: rebuilt and synchronized wikiversions files: group0 to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>19:13 jayme@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-ctrl1001.eqiad.wmnet with reason: host reimage</li> <li>19:06 brennen: 1.44.0-wmf.3 train status (<a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a>): no current blockers, rolling to group0.</li> <li>18:55 moritzm: installing libarchive security updates</li> <li>18:55 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>18:31 swfrench@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087604" class="extiw" title="gerrit:1087604">Add title-case mapping to support migration to PHP 8.1 (T372603)</a> (duration: 18m 48s)</li> <li>18:25 swfrench@deploy2002: swfrench: Continuing with sync</li> <li>18:24 swfrench-wmf: verified consistent 7.4-like title-case behavior in 7.4- and 8.1-based images, verified expected treatment of eszett in mwdebug - <a href="https://phabricator.wikimedia.org/T372603" class="extiw" title="phab:T372603">T372603</a></li> <li>18:19 swfrench@deploy2002: swfrench: Backport for <a href="https://gerrit.wikimedia.org/r/1087604" class="extiw" title="gerrit:1087604">Add title-case mapping to support migration to PHP 8.1 (T372603)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>18:12 swfrench@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087604" class="extiw" title="gerrit:1087604">Add title-case mapping to support migration to PHP 8.1 (T372603)</a></li> <li>18:08 jayme@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>18:01 moritzm: remove ganeti1012 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>17:59 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:57 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:57 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:56 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:35 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:34 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:26 brennen@deploy2002: Finished scap sync-world: testwikis to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a> (duration: 45m 29s)</li> <li>16:55 jgiannelos@deploy2002: helmfile [codfw] DONE helmfile.d/services/push-notifications: apply</li> <li>16:54 jgiannelos@deploy2002: helmfile [codfw] START helmfile.d/services/push-notifications: apply</li> <li>16:54 jgiannelos@deploy2002: helmfile [eqiad] DONE helmfile.d/services/push-notifications: apply</li> <li>16:53 jgiannelos@deploy2002: helmfile [eqiad] START helmfile.d/services/push-notifications: apply</li> <li>16:48 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>16:47 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-ctrl1001.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL</li> <li>16:40 brennen@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>16:39 jayme@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-ctrl1001.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL</li> <li>16:37 jayme@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>16:34 dancy@deploy2002: Installation of scap version "4.123.0" completed for 209 hosts</li> <li>16:30 dancy@deploy2002: Installing scap version "4.123.0" for 209 hosts</li> <li>16:18 jgiannelos@deploy2002: helmfile [eqiad] DONE helmfile.d/services/push-notifications: apply</li> <li>16:18 jgiannelos@deploy2002: helmfile [eqiad] START helmfile.d/services/push-notifications: apply</li> <li>16:17 jgiannelos@deploy2002: helmfile [codfw] DONE helmfile.d/services/push-notifications: apply</li> <li>16:17 jgiannelos@deploy2002: helmfile [codfw] START helmfile.d/services/push-notifications: apply</li> <li>16:16 jgiannelos@deploy2002: helmfile [staging] DONE helmfile.d/services/push-notifications: apply</li> <li>16:15 jgiannelos@deploy2002: helmfile [staging] START helmfile.d/services/push-notifications: apply</li> <li>16:13 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for cr[1-2]-eqiad</li> <li>16:13 cmooney@cumin1002: START - Cookbook sre.hosts.remove-downtime for cr[1-2]-eqiad</li> <li>16:08 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>16:07 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>15:57 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>15:56 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>15:55 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>15:52 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>15:52 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>15:47 jayme@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>15:42 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>15:42 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update dns records for IPs moving from old to new fundraising firewalls - cmooney@cumin1002"</li> <li>15:35 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update dns records for IPs moving from old to new fundraising firewalls - cmooney@cumin1002"</li> <li>15:27 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>15:19 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>15:16 jayme@cumin2002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for wikikube-ctrl1002.eqiad.wmnet</li> <li>15:16 jayme@cumin2002: START - Cookbook sre.hosts.remove-downtime for wikikube-ctrl1002.eqiad.wmnet</li> <li>15:16 topranks: moving fundraising links in eqiad from old to new firewall cluster and switches (<a href="https://phabricator.wikimedia.org/T377381" class="extiw" title="phab:T377381">T377381</a>)</li> <li>15:14 jayme@cumin2002: START - Cookbook sre.k8s.reimage-stacked-control-plane Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>15:13 jayme@cumin2002: END (FAIL) - Cookbook sre.k8s.reimage-stacked-control-plane (exit_code=99) Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>15:10 jayme@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>15:04 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on cr[1-2]-eqiad,pfw3-eqiad with reason: fundraising tech migration to new equipment</li> <li>15:04 cmooney@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on cr[1-2]-eqiad,pfw3-eqiad with reason: fundraising tech migration to new equipment</li> <li>15:02 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>14:30 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on fasw-c-eqiad with reason: fundraising tech migration to new equipment</li> <li>14:30 cmooney@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on fasw-c-eqiad with reason: fundraising tech migration to new equipment</li> <li>14:28 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>14:28 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update dns records for IPs moving from old to new fundraising firewalls - cmooney@cumin1002"</li> <li>14:28 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update dns records for IPs moving from old to new fundraising firewalls - cmooney@cumin1002"</li> <li>14:26 moritzm: installing apache2 security updates</li> <li>14:23 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>14:08 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>14:08 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>14:03 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090455" class="extiw" title="gerrit:1090455">[CirrusSearch] testwiki: enable offloading weighted tags via EventBus (T378983)</a></li> <li>13:58 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090455" class="extiw" title="gerrit:1090455">[CirrusSearch] testwiki: enable offloading weighted tags via EventBus (T378983)</a></li> <li>13:48 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>13:47 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>13:43 jnuche@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>13:37 jnuche@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.3 refs <a href="https://phabricator.wikimedia.org/T375662" class="extiw" title="phab:T375662">T375662</a></li> <li>13:21 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>13:15 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of dse-k8s-etcd1003.eqiad.wmnet to plain</li> <li>13:14 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of dse-k8s-etcd1003.eqiad.wmnet to plain</li> <li>13:11 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>13:11 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>13:10 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:10 jayme@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-ctrl1001.eqiad.wmnet with OS bookworm</li> <li>13:09 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of dse-k8s-etcd1003.eqiad.wmnet to drbd</li> <li>13:09 jayme@cumin2002: START - Cookbook sre.k8s.reimage-stacked-control-plane Reimaging k8s control planes of cluster wikikube-eqiad: containerd migration</li> <li>13:09 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>12:59 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of dse-k8s-etcd1003.eqiad.wmnet to drbd</li> <li>12:54 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ml-etcd1002.eqiad.wmnet to plain</li> <li>12:53 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ml-etcd1002.eqiad.wmnet to plain</li> <li>12:53 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>12:52 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>12:45 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ml-etcd1002.eqiad.wmnet to drbd</li> <li>12:35 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ml-etcd1002.eqiad.wmnet to drbd</li> <li>12:28 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>12:28 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db2236 slowly with 10 steps - slow repool <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>12:25 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1012.eqiad.wmnet</li> <li>12:09 moritzm: remove ganeti1015 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>12:08 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti1010.eqiad.wmnet</li> <li>12:08 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>12:08 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1010.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>12:04 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1010.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>11:54 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1015.eqiad.wmnet</li> <li>11:54 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>11:52 elukey@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>11:48 fabfur@cumin1002: conftool action : set/pooled=no; selector: name=cp5017.eqsin.wmnet</li> <li>11:47 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti1010.eqiad.wmnet</li> <li>11:42 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti1013.eqiad.wmnet</li> <li>11:42 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>11:42 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1013.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>11:40 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti1013.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>11:37 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>11:27 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti1013.eqiad.wmnet</li> <li>11:23 btullis@cumin1002: END (PASS) - Cookbook sre.druid.roll-restart-workers (exit_code=0) for Druid analytics cluster: Roll restart of Druid jvm daemons.</li> <li>11:01 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>11:01 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>10:45 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db2217 gradually with 4 steps - <a href="https://phabricator.wikimedia.org/T379491" class="extiw" title="phab:T379491">T379491</a></li> <li>10:37 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>10:37 btullis@cumin1002: START - Cookbook sre.druid.roll-restart-workers for Druid analytics cluster: Roll restart of Druid jvm daemons.</li> <li>10:36 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>10:36 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>10:36 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>10:12 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db2236 slowly with 10 steps - slow repool <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>09:59 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db2217 gradually with 4 steps - <a href="https://phabricator.wikimedia.org/T379491" class="extiw" title="phab:T379491">T379491</a></li> <li>09:48 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71006">https://phabricator.wikimedia.org/P71006</a> and previous config saved to /var/cache/conftool/dbconfig/20241112-094851-arnaudb.json</li> <li>09:41 moritzm: update d-i netboot image for 12.8 point release <a href="https://phabricator.wikimedia.org/T379600" class="extiw" title="phab:T379600">T379600</a></li> <li>09:33 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71005">https://phabricator.wikimedia.org/P71005</a> and previous config saved to /var/cache/conftool/dbconfig/20241112-093343-arnaudb.json</li> <li>09:18 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090428" class="extiw" title="gerrit:1090428">Revert "CirrusSearch: re-enable offloading weighted tags via EventBus"</a> (duration: 06m 46s)</li> <li>09:18 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71004">https://phabricator.wikimedia.org/P71004</a> and previous config saved to /var/cache/conftool/dbconfig/20241112-091836-arnaudb.json</li> <li>09:17 elukey@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>09:14 urbanecm@deploy2002: trainbranchbot, urbanecm: Continuing with sync</li> <li>09:14 urbanecm@deploy2002: trainbranchbot, urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1090428" class="extiw" title="gerrit:1090428">Revert "CirrusSearch: re-enable offloading weighted tags via EventBus"</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>09:11 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1090428" class="extiw" title="gerrit:1090428">Revert "CirrusSearch: re-enable offloading weighted tags via EventBus"</a></li> <li>09:10 urbanecm@deploy2002: Sync cancelled.</li> <li>09:03 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71002">https://phabricator.wikimedia.org/P71002</a> and previous config saved to /var/cache/conftool/dbconfig/20241112-090329-arnaudb.json</li> <li>08:38 urbanecm@deploy2002: pfischer, urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1089826" class="extiw" title="gerrit:1089826">CirrusSearch: re-enable offloading weighted tags via EventBus (T378983)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:36 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089826" class="extiw" title="gerrit:1089826">CirrusSearch: re-enable offloading weighted tags via EventBus (T378983)</a></li> <li>08:32 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1015.eqiad.wmnet</li> <li>08:31 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1015.eqiad.wmnet</li> <li>08:28 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089230" class="extiw" title="gerrit:1089230">Fix WeightedTagsUpdater (T378664 T378983)</a> (duration: 06m 59s)</li> <li>08:25 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1015.eqiad.wmnet</li> <li>08:21 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089230" class="extiw" title="gerrit:1089230">Fix WeightedTagsUpdater (T378664 T378983)</a></li> <li>08:19 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1009.eqiad.wmnet</li> <li>08:17 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1009.eqiad.wmnet</li> <li>08:04 moritzm: installing apache security updates</li> <li>08:03 arnaudb@cumin1002: dbctl commit (dc=all): 'Depooling db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P71001">https://phabricator.wikimedia.org/P71001</a> and previous config saved to /var/cache/conftool/dbconfig/20241112-080303-arnaudb.json</li> <li>08:02 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2187.codfw.wmnet with reason: Maintenance</li> <li>08:02 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 8:00:00 on db2187.codfw.wmnet with reason: Maintenance</li> <li>08:02 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2155.codfw.wmnet with reason: Maintenance</li> <li>08:02 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on db2155.codfw.wmnet with reason: Maintenance</li> <li>07:53 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti-test2003</li> <li>07:53 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti-test2003</li> <li>07:52 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:52 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>05:01 mwpresync@deploy2002: Pruned MediaWiki: 1.43.0-wmf.28 (duration: 01m 52s)</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-11" data-mw-thread-id="h-2024-11-11"><span data-mw-comment-start="" id="h-2024-11-11"></span>2024-11-11<span data-mw-comment-end="h-2024-11-11"></span></h2></div> <ul><li>away: UTC late deploys done</li> <li>23:08 tgr@deploy2002: scap failed: <CalledProcessError> Command '['sudo', '-u', 'mwbuilder', '-n', '--', '/usr/bin/scap', 'mwscript', '--no-local-config', '--directory', '/srv/mediawiki-staging', '--user', 'www-data', '--network', '--', 'purgeMessageBlobStore.php']' returned non-zero exit status 1. (scap version: 4.122.0) (duration: 11m 44s)</li> <li>23:02 tgr@deploy2002: d3r1ck01, tgr: Continuing with sync</li> <li>22:59 tgr@deploy2002: d3r1ck01, tgr: Backport for <a href="https://gerrit.wikimedia.org/r/1089807" class="extiw" title="gerrit:1089807">PageUpdater: restore call to RevisionFromEditComplete (T379152)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:56 tgr@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089807" class="extiw" title="gerrit:1089807">PageUpdater: restore call to RevisionFromEditComplete (T379152)</a></li> <li>22:30 tgr@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089864" class="extiw" title="gerrit:1089864">contactpage: Update AffCom contact form messages (Resubmit) (T375392)</a> (duration: 25m 48s)</li> <li>22:21 tgr@deploy2002: tgr: Continuing with sync</li> <li>22:19 tgr@deploy2002: tgr: Backport for <a href="https://gerrit.wikimedia.org/r/1089864" class="extiw" title="gerrit:1089864">contactpage: Update AffCom contact form messages (Resubmit) (T375392)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>22:13 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="4330588d"><a class="external text" href="https://gerrit.wikimedia.org/r/q/4330588d">4330588d</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="bcd072a1"><a class="external text" href="https://gerrit.wikimedia.org/r/q/bcd072a1">bcd072a1</a></span></li> <li>22:05 tgr@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089864" class="extiw" title="gerrit:1089864">contactpage: Update AffCom contact form messages (Resubmit) (T375392)</a></li> <li>21:38 tgr@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1082174" class="extiw" title="gerrit:1082174">contactpages: Update Affcom UserGroup application form (T375392)</a> (duration: 28m 07s)</li> <li>21:33 tgr@deploy2002: ammarpad, tgr: Continuing with sync</li> <li>21:12 tgr@deploy2002: ammarpad, tgr: Backport for <a href="https://gerrit.wikimedia.org/r/1082174" class="extiw" title="gerrit:1082174">contactpages: Update Affcom UserGroup application form (T375392)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:10 tgr@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1082174" class="extiw" title="gerrit:1082174">contactpages: Update Affcom UserGroup application form (T375392)</a></li> <li>20:21 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="65a8de90"><a class="external text" href="https://gerrit.wikimedia.org/r/q/65a8de90">65a8de90</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="4330588d"><a class="external text" href="https://gerrit.wikimedia.org/r/q/4330588d">4330588d</a></span></li> <li>17:55 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Add superset links - oblivian@cumin1002 - <a href="https://phabricator.wikimedia.org/T379567" class="extiw" title="phab:T379567">T379567</a>"</li> <li>17:55 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Add superset links - oblivian@cumin1002 - <a href="https://phabricator.wikimedia.org/T379567" class="extiw" title="phab:T379567">T379567</a></li> <li>17:54 oblivian@cumin1002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Add superset links - oblivian@cumin1002 - <a href="https://phabricator.wikimedia.org/T379567" class="extiw" title="phab:T379567">T379567</a></li> <li>17:54 oblivian@cumin1002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Add superset links - oblivian@cumin1002 - <a href="https://phabricator.wikimedia.org/T379567" class="extiw" title="phab:T379567">T379567</a>"</li> <li>16:19 elukey: restart pybal on lvs2013 (primary) to pick up new kartotherian-k8s-ssl service</li> <li>16:17 elukey: restart pybal on lvs2014 (secondary) to pick up new kartotherian-k8s-ssl service</li> <li>16:10 elukey: restart pybal on lvs1019 (primary) to pick up new kartotherian-k8s-ssl service</li> <li>16:09 elukey: restart pybal on lvs1020 (secondary) to pick up new kartotherian-k8s-ssl service</li> <li>16:09 moritzm: installing libarchive security updates</li> <li>15:55 elukey@puppetserver1001: conftool action : set/pooled=yes:weight=10; selector: dc=codfw,cluster=maps,service=kartotherian-k8s-ssl</li> <li>15:55 elukey@puppetserver1001: conftool action : set/pooled=yes:weight=10; selector: dc=eqiad,cluster=maps,service=kartotherian-k8s-ssl</li> <li>15:54 elukey@puppetserver1001: conftool action : set/pooled=yes:weight=1; selector: cluster=codfw,service=kartotherian-k8s-ssl</li> <li>15:04 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1311.eqiad.wmnet with OS bookworm</li> <li>15:04 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>15:04 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>15:03 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1309.eqiad.wmnet with OS bookworm</li> <li>15:03 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>15:00 Lucas_WMDE: UTC afternoon backport+config window done</li> <li>15:00 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089739" class="extiw" title="gerrit:1089739">wikipedias: clear link-recommendations on page save (T379522)</a> (duration: 10m 59s)</li> <li>14:58 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:56 lucaswerkmeister-wmde@deploy2002: migr, lucaswerkmeister-wmde: Continuing with sync</li> <li>14:51 lucaswerkmeister-wmde@deploy2002: migr, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1089739" class="extiw" title="gerrit:1089739">wikipedias: clear link-recommendations on page save (T379522)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:49 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089739" class="extiw" title="gerrit:1089739">wikipedias: clear link-recommendations on page save (T379522)</a></li> <li>14:44 btullis@cumin1002: END (FAIL) - Cookbook sre.presto.roll-restart-workers (exit_code=99) for Presto an-presto cluster: Roll restart of all Presto's jvm daemons.</li> <li>14:37 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1310.eqiad.wmnet with OS bookworm</li> <li>14:37 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:36 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:35 elukey@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>14:33 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1312.eqiad.wmnet with OS bookworm</li> <li>14:33 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:32 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:32 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1306.eqiad.wmnet with OS bookworm</li> <li>14:32 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:32 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:28 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1308.eqiad.wmnet with OS bookworm</li> <li>14:28 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:28 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:27 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2088.codfw.wmnet with OS bullseye</li> <li>14:27 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1309.eqiad.wmnet with reason: host reimage</li> <li>14:26 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1307.eqiad.wmnet with OS bookworm</li> <li>14:26 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:25 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:22 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1311.eqiad.wmnet with reason: host reimage</li> <li>14:22 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker1305.eqiad.wmnet with OS bookworm</li> <li>14:22 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:21 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>14:20 zabe@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1078764" class="extiw" title="gerrit:1078764">zhwiki: Allow event-organizer self remove usergroup (T376061)</a> (duration: 10m 40s)</li> <li>14:20 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2088.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>14:19 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1310.eqiad.wmnet with reason: host reimage</li> <li>14:16 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1306.eqiad.wmnet with reason: host reimage</li> <li>14:15 zabe@deploy2002: zabe, zhaofjx: Continuing with sync</li> <li>14:13 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1312.eqiad.wmnet with reason: host reimage</li> <li>14:12 zabe@deploy2002: zabe, zhaofjx: Backport for <a href="https://gerrit.wikimedia.org/r/1078764" class="extiw" title="gerrit:1078764">zhwiki: Allow event-organizer self remove usergroup (T376061)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:10 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1308.eqiad.wmnet with reason: host reimage</li> <li>14:09 zabe@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1078764" class="extiw" title="gerrit:1078764">zhwiki: Allow event-organizer self remove usergroup (T376061)</a></li> <li>14:07 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2088.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>14:07 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1307.eqiad.wmnet with reason: host reimage</li> <li>14:06 btullis@cumin1002: START - Cookbook sre.presto.roll-restart-workers for Presto an-presto cluster: Roll restart of all Presto's jvm daemons.</li> <li>14:05 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts irc2002.wikimedia.org</li> <li>14:05 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>14:05 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: irc2002.wikimedia.org decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>14:05 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: irc2002.wikimedia.org decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>14:04 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1312.eqiad.wmnet with reason: host reimage</li> <li>14:04 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1308.eqiad.wmnet with reason: host reimage</li> <li>14:04 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1309.eqiad.wmnet with reason: host reimage</li> <li>14:04 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1311.eqiad.wmnet with reason: host reimage</li> <li>14:04 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker1305.eqiad.wmnet with reason: host reimage</li> <li>14:04 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1310.eqiad.wmnet with reason: host reimage</li> <li>14:03 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1307.eqiad.wmnet with reason: host reimage</li> <li>14:03 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1306.eqiad.wmnet with reason: host reimage</li> <li>14:00 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker1305.eqiad.wmnet with reason: host reimage</li> <li>13:55 moritzm: powercycled ganeti2031</li> <li>13:44 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>13:39 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts irc2002.wikimedia.org</li> <li>13:38 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts irc1002.wikimedia.org</li> <li>13:38 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>13:38 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: irc1002.wikimedia.org decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>13:34 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1312.eqiad.wmnet with OS bookworm</li> <li>13:34 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1311.eqiad.wmnet with OS bookworm</li> <li>13:34 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: irc1002.wikimedia.org decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>13:34 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1311.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:33 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1312.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:33 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1310.eqiad.wmnet with OS bookworm</li> <li>13:32 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1309.eqiad.wmnet with OS bookworm</li> <li>13:32 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1308.eqiad.wmnet with OS bookworm</li> <li>13:32 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1307.eqiad.wmnet with OS bookworm</li> <li>13:32 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1306.eqiad.wmnet with OS bookworm</li> <li>13:31 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1306.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:31 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host wikikube-worker1305.eqiad.wmnet with OS bookworm</li> <li>13:30 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>13:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1307.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1309.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1310.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1308.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:29 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1305.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:25 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts irc1002.wikimedia.org</li> <li>13:22 jynus: reverting deleted rows on db1176 (mailman3) <a href="https://phabricator.wikimedia.org/T379519" class="extiw" title="phab:T379519">T379519</a></li> <li>13:16 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1312.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:15 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1311.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:12 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1050.eqiad.wmnet to cluster eqiad and group D</li> <li>13:12 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1306.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:11 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1050.eqiad.wmnet to cluster eqiad and group D</li> <li>13:11 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1310.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:11 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host wikikube-worker1306.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:11 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1309.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:11 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1308.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:11 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1307.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:10 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1306.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:10 jclark@cumin1002: START - Cookbook sre.hosts.provision for host wikikube-worker1305.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>13:10 dreamyjazz@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085593" class="extiw" title="gerrit:1085593">Exclude temp account viewer autopromotions from RC (T377829)</a> (duration: 07m 07s)</li> <li>13:08 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>13:08 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>13:08 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for wikikube-worker - jclark@cumin1002"</li> <li>13:05 dreamyjazz@deploy2002: mszabo, dreamyjazz: Continuing with sync</li> <li>13:05 dreamyjazz@deploy2002: mszabo, dreamyjazz: Backport for <a href="https://gerrit.wikimedia.org/r/1085593" class="extiw" title="gerrit:1085593">Exclude temp account viewer autopromotions from RC (T377829)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>13:05 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Fix bug in requestctl commit - oblivian@cumin1002"</li> <li>13:05 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Fix bug in requestctl commit - oblivian@cumin1002</li> <li>13:04 oblivian@cumin1002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Fix bug in requestctl commit - oblivian@cumin1002</li> <li>13:04 oblivian@cumin1002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Fix bug in requestctl commit - oblivian@cumin1002"</li> <li>13:04 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>13:03 dreamyjazz@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085593" class="extiw" title="gerrit:1085593">Exclude temp account viewer autopromotions from RC (T377829)</a></li> <li>13:00 btullis@cumin1002: END (PASS) - Cookbook sre.zookeeper.roll-restart-zookeeper (exit_code=0) for Zookeeper A:zookeeper-druid-analytics cluster: Roll restart of jvm daemons.</li> <li>12:54 btullis@cumin1002: START - Cookbook sre.zookeeper.roll-restart-zookeeper for Zookeeper A:zookeeper-druid-analytics cluster: Roll restart of jvm daemons.</li> <li>12:48 btullis@cumin1002: END (PASS) - Cookbook sre.zookeeper.roll-restart-zookeeper (exit_code=0) for Zookeeper A:zookeeper-druid-public cluster: Roll restart of jvm daemons.</li> <li>12:42 btullis@cumin1002: START - Cookbook sre.zookeeper.roll-restart-zookeeper for Zookeeper A:zookeeper-druid-public cluster: Roll restart of jvm daemons.</li> <li>12:41 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1049.eqiad.wmnet to cluster eqiad and group D</li> <li>12:40 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1049.eqiad.wmnet to cluster eqiad and group D</li> <li>12:36 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1050.eqiad.wmnet</li> <li>12:29 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1050.eqiad.wmnet</li> <li>12:28 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1049.eqiad.wmnet</li> <li>12:23 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>12:21 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1049.eqiad.wmnet</li> <li>12:18 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1050</li> <li>12:16 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1050</li> <li>12:16 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1049</li> <li>12:15 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1049</li> <li>12:13 btullis@cumin1002: END (PASS) - Cookbook sre.zookeeper.roll-restart-zookeeper (exit_code=0) for Zookeeper A:zookeeper-analytics cluster: Roll restart of jvm daemons.</li> <li>12:06 btullis@cumin1002: START - Cookbook sre.zookeeper.roll-restart-zookeeper for Zookeeper A:zookeeper-analytics cluster: Roll restart of jvm daemons.</li> <li>12:01 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2083.codfw.wmnet with reason: host reimage</li> <li>11:56 elukey@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2083.codfw.wmnet with reason: host reimage</li> <li>11:56 btullis@cumin1002: END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host an-redacteddb1001.eqiad.wmnet</li> <li>11:54 btullis@cumin1002: END (PASS) - Cookbook sre.opensearch.roll-restart-reboot (exit_code=0) rolling restart_daemons on A:datahubsearch</li> <li>11:46 btullis@cumin1002: START - Cookbook sre.opensearch.roll-restart-reboot rolling restart_daemons on A:datahubsearch</li> <li>11:44 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>11:43 btullis@cumin1002: START - Cookbook sre.hosts.reboot-single for host an-redacteddb1001.eqiad.wmnet</li> <li>11:43 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>11:43 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>11:30 elukey@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>11:06 elukey@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>11:04 btullis@cumin1002: END (PASS) - Cookbook sre.wikireplicas.update-views (exit_code=0)</li> <li>10:57 btullis@cumin1002: START - Cookbook sre.wikireplicas.update-views</li> <li>10:55 elukey@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li> <li>10:01 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Update to latest - oblivian@cumin1002"</li> <li>10:01 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Update to latest - oblivian@cumin1002</li> <li>10:00 oblivian@cumin1002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Update to latest - oblivian@cumin1002</li> <li>10:00 oblivian@cumin1002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Update to latest - oblivian@cumin1002"</li> <li>09:10 moritzm: remove ganeti1011 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>09:02 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1011.eqiad.wmnet</li> <li>08:40 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088628" class="extiw" title="gerrit:1088628">Update Wikimedia Foundation primary address. (T379417)</a>, <a href="https://gerrit.wikimedia.org/r/1082559" class="extiw" title="gerrit:1082559">Update Office Wiki favicon to use wmf.ico and also delete now unused office.ico file. (T378026)</a> (duration: 07m 15s)</li> <li>08:35 urbanecm@deploy2002: urbanecm, varnent: Continuing with sync</li> <li>08:35 urbanecm@deploy2002: urbanecm, varnent: Backport for <a href="https://gerrit.wikimedia.org/r/1088628" class="extiw" title="gerrit:1088628">Update Wikimedia Foundation primary address. (T379417)</a>, <a href="https://gerrit.wikimedia.org/r/1082559" class="extiw" title="gerrit:1082559">Update Office Wiki favicon to use wmf.ico and also delete now unused office.ico file. (T378026)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:32 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088628" class="extiw" title="gerrit:1088628">Update Wikimedia Foundation primary address. (T379417)</a>, <a href="https://gerrit.wikimedia.org/r/1082559" class="extiw" title="gerrit:1082559">Update Office Wiki favicon to use wmf.ico and also delete now unused office.ico file. (T378026)</a></li> <li>08:32 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089182" class="extiw" title="gerrit:1089182">Allow wgGroupsRemoveFromSelf for templateeditor, confirmed, and abusefilter-helper in zhwiki (T379500)</a> (duration: 20m 59s)</li> <li>08:24 urbanecm@deploy2002: urbanecm, hamishz: Continuing with sync</li> <li>08:22 urbanecm@deploy2002: urbanecm, hamishz: Backport for <a href="https://gerrit.wikimedia.org/r/1089182" class="extiw" title="gerrit:1089182">Allow wgGroupsRemoveFromSelf for templateeditor, confirmed, and abusefilter-helper in zhwiki (T379500)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:18 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Update to latest - oblivian@cumin1002"</li> <li>08:18 oblivian@cumin1002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Update to latest - oblivian@cumin1002</li> <li>08:17 oblivian@cumin1002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Update to latest - oblivian@cumin1002</li> <li>08:17 oblivian@cumin1002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Update to latest - oblivian@cumin1002"</li> <li>08:11 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1089182" class="extiw" title="gerrit:1089182">Allow wgGroupsRemoveFromSelf for templateeditor, confirmed, and abusefilter-helper in zhwiki (T379500)</a></li> <li>07:51 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1011.eqiad.wmnet</li> <li>07:49 _joe_: installing conftool 4.1.0 on puppetservers</li> <li>07:15 kartik@deploy2002: helmfile [ml-staging-codfw] 'sync' command on namespace 'recommendation-api-ng' for release 'main' .</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-10" data-mw-thread-id="h-2024-11-10"><span data-mw-comment-start="" id="h-2024-11-10"></span>2024-11-10<span data-mw-comment-end="h-2024-11-10"></span></h2></div> <ul><li>23:43 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>23:17 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>23:14 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:51 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>22:29 jhathaway: re-imaging ms-be2082 to test efi boot order</li> <li>12:32 elukey: optimize table `archive` on db2217 - frwiki db - corrupt index error (host already depooled)</li> <li>12:26 slyngshede@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on db2217.codfw.wmnet with reason: Corrupt Index</li> <li>12:26 slyngshede@cumin1002: START - Cookbook sre.hosts.downtime for 3 days, 0:00:00 on db2217.codfw.wmnet with reason: Corrupt Index</li> <li>12:25 slyngshede@cumin1002: dbctl commit (dc=all): 'Depool db2217', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70997">https://phabricator.wikimedia.org/P70997</a> and previous config saved to /var/cache/conftool/dbconfig/20241110-122532-slyngshede.json</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-09" data-mw-thread-id="h-2024-11-09"><span data-mw-comment-start="" id="h-2024-11-09"></span>2024-11-09<span data-mw-comment-end="h-2024-11-09"></span></h2></div> <ul><li>14:49 dani@deploy2002: helmfile [codfw] DONE helmfile.d/services/miscweb: apply</li> <li>14:49 dani@deploy2002: helmfile [codfw] START helmfile.d/services/miscweb: apply</li> <li>14:48 dani@deploy2002: helmfile [eqiad] DONE helmfile.d/services/miscweb: apply</li> <li>14:48 dani@deploy2002: helmfile [eqiad] START helmfile.d/services/miscweb: apply</li> <li>14:48 dani@deploy2002: helmfile [staging] DONE helmfile.d/services/miscweb: apply</li> <li>14:48 dani@deploy2002: helmfile [staging] START helmfile.d/services/miscweb: apply</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-08" data-mw-thread-id="h-2024-11-08"><span data-mw-comment-start="" id="h-2024-11-08"></span>2024-11-08<span data-mw-comment-end="h-2024-11-08"></span></h2></div> <ul><li>23:35 zabe: attach Sotiale's local accounts on newly created wikis</li> <li>23:16 Reedy: ran `delete from oathauth_devices where oad_id=4506;` on centralauth for <a href="https://phabricator.wikimedia.org/T379398" class="extiw" title="phab:T379398">T379398</a> because oad_user=0</li> <li>23:07 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>22:54 dani@deploy2002: helmfile [codfw] DONE helmfile.d/services/miscweb: apply</li> <li>22:54 dani@deploy2002: helmfile [codfw] START helmfile.d/services/miscweb: apply</li> <li>22:54 dani@deploy2002: helmfile [eqiad] DONE helmfile.d/services/miscweb: apply</li> <li>22:54 dani@deploy2002: helmfile [eqiad] START helmfile.d/services/miscweb: apply</li> <li>22:54 dani@deploy2002: helmfile [staging] DONE helmfile.d/services/miscweb: apply</li> <li>22:54 dani@deploy2002: helmfile [staging] START helmfile.d/services/miscweb: apply</li> <li>22:52 dani@deploy2002: helmfile [codfw] DONE helmfile.d/services/miscweb: apply</li> <li>22:51 dani@deploy2002: helmfile [codfw] START helmfile.d/services/miscweb: apply</li> <li>22:51 dani@deploy2002: helmfile [eqiad] DONE helmfile.d/services/miscweb: apply</li> <li>22:51 dani@deploy2002: helmfile [eqiad] START helmfile.d/services/miscweb: apply</li> <li>22:51 dani@deploy2002: helmfile [staging] DONE helmfile.d/services/miscweb: apply</li> <li>22:51 dani@deploy2002: helmfile [staging] START helmfile.d/services/miscweb: apply</li> <li>22:44 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:41 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:39 dani@deploy2002: helmfile [codfw] DONE helmfile.d/services/miscweb: apply</li> <li>22:39 dani@deploy2002: helmfile [codfw] START helmfile.d/services/miscweb: apply</li> <li>22:39 dani@deploy2002: helmfile [eqiad] DONE helmfile.d/services/miscweb: apply</li> <li>22:38 dani@deploy2002: helmfile [eqiad] START helmfile.d/services/miscweb: apply</li> <li>22:38 dani@deploy2002: helmfile [staging] DONE helmfile.d/services/miscweb: apply</li> <li>22:38 dani@deploy2002: helmfile [staging] START helmfile.d/services/miscweb: apply</li> <li>22:29 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>22:28 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>22:08 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>21:18 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:18 denisse: disabling Puppet on grafana2001 - <a href="https://phabricator.wikimedia.org/T379043" class="extiw" title="phab:T379043">T379043</a></li> <li>21:17 jhathaway@cumin2002: START - Cookbook sre.hosts.provision for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:12 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>21:08 mutante: cumint2002 [cumin2002:~] $ sudo systemctl reset-failed</li> <li>21:05 mutante: cumin2002 - sudo systemctl status httpbb_kubernetes_mw-api-int_hourly</li> <li>20:28 aude@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088586" class="extiw" title="gerrit:1088586">Reviving "Update interwiki map"</a> (duration: 10m 19s)</li> <li>20:24 aude@deploy2002: seddon, aude: Continuing with sync</li> <li>20:21 aude@deploy2002: seddon, aude: Backport for <a href="https://gerrit.wikimedia.org/r/1088586" class="extiw" title="gerrit:1088586">Reviving "Update interwiki map"</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>20:20 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>20:18 aude@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088586" class="extiw" title="gerrit:1088586">Reviving "Update interwiki map"</a></li> <li>20:15 aude@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088375" class="extiw" title="gerrit:1088375">Enable Tabular data for test commons (T378127)</a> (duration: 10m 55s)</li> <li>20:10 aude@deploy2002: aude: Continuing with sync</li> <li>20:06 aude@deploy2002: aude: Backport for <a href="https://gerrit.wikimedia.org/r/1088375" class="extiw" title="gerrit:1088375">Enable Tabular data for test commons (T378127)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>20:04 aude@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088375" class="extiw" title="gerrit:1088375">Enable Tabular data for test commons (T378127)</a></li> <li>20:02 aude@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088366" class="extiw" title="gerrit:1088366">Reopen testcommonswiki for testing Chart extension</a> (duration: 14m 33s)</li> <li>19:59 jhathaway@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>19:59 jhathaway@cumin1002: START - Cookbook sre.hosts.downtime for 1:00:00 on ms-be2082.codfw.wmnet with reason: <a href="https://phabricator.wikimedia.org/T371400" class="extiw" title="phab:T371400">T371400</a></li> <li>19:57 aude@deploy2002: aude: Continuing with sync</li> <li>19:50 aude@deploy2002: aude: Backport for <a href="https://gerrit.wikimedia.org/r/1088366" class="extiw" title="gerrit:1088366">Reopen testcommonswiki for testing Chart extension</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>19:47 aude@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088366" class="extiw" title="gerrit:1088366">Reopen testcommonswiki for testing Chart extension</a></li> <li>18:40 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2168.codfw.wmnet with OS bookworm</li> <li>18:40 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:40 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>18:39 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2167.codfw.wmnet with OS bookworm</li> <li>18:38 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:37 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2170.codfw.wmnet with OS bookworm</li> <li>18:33 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:32 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:31 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2169.codfw.wmnet with OS bookworm</li> <li>18:31 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:29 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2166.codfw.wmnet with OS bookworm</li> <li>18:27 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:27 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:26 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2165.codfw.wmnet with OS bookworm</li> <li>18:26 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:23 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:21 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>18:21 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Create new snippets for frack IPs - cmooney@cumin1002"</li> <li>18:21 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Create new snippets for frack IPs - cmooney@cumin1002"</li> <li>18:21 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2164.codfw.wmnet with OS bookworm</li> <li>18:21 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:20 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2168.codfw.wmnet with reason: host reimage</li> <li>18:19 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:17 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:17 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2167.codfw.wmnet with reason: host reimage</li> <li>18:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2170.codfw.wmnet with reason: host reimage</li> <li>18:10 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2169.codfw.wmnet with reason: host reimage</li> <li>18:10 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2170.codfw.wmnet with reason: host reimage</li> <li>18:07 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2166.codfw.wmnet with reason: host reimage</li> <li>18:06 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2169.codfw.wmnet with reason: host reimage</li> <li>18:04 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2165.codfw.wmnet with reason: host reimage</li> <li>18:03 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2168.codfw.wmnet with reason: host reimage</li> <li>18:01 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2167.codfw.wmnet with reason: host reimage</li> <li>18:01 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2164.codfw.wmnet with reason: host reimage</li> <li>17:59 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2145.codfw.wmnet with OS bookworm</li> <li>17:59 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:59 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:59 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2166.codfw.wmnet with reason: host reimage</li> <li>17:57 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2165.codfw.wmnet with reason: host reimage</li> <li>17:57 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:57 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Create new snippets for frack IPs - cmooney@cumin1002"</li> <li>17:56 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Create new snippets for frack IPs - cmooney@cumin1002"</li> <li>17:56 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2144.codfw.wmnet with OS bookworm</li> <li>17:56 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:56 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>17:56 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:56 herron@cumin1002: END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host aux-k8s-worker1005.eqiad.wmnet</li> <li>17:56 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-worker1005.eqiad.wmnet with OS bookworm</li> <li>17:56 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2164.codfw.wmnet with reason: host reimage</li> <li>17:54 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:52 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>17:50 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2170.codfw.wmnet with OS bookworm</li> <li>17:50 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2157.codfw.wmnet with OS bookworm</li> <li>17:50 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:49 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:49 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:47 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2169.codfw.wmnet with OS bookworm</li> <li>17:46 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2160.codfw.wmnet with OS bookworm</li> <li>17:46 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:45 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:44 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2168.codfw.wmnet with OS bookworm</li> <li>17:44 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2158.codfw.wmnet with OS bookworm</li> <li>17:44 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:43 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2167.codfw.wmnet with OS bookworm</li> <li>17:42 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2162.codfw.wmnet with OS bookworm</li> <li>17:42 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:40 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2166.codfw.wmnet with OS bookworm</li> <li>17:40 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2145.codfw.wmnet with reason: host reimage</li> <li>17:40 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2156.codfw.wmnet with OS bookworm</li> <li>17:39 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2165.codfw.wmnet with OS bookworm</li> <li>17:38 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2161.codfw.wmnet with OS bookworm</li> <li>17:38 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:37 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on wikikube-worker2144.codfw.wmnet with reason: host reimage</li> <li>17:37 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2164.codfw.wmnet with OS bookworm</li> <li>17:37 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-worker1005.eqiad.wmnet with reason: host reimage</li> <li>17:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2159.codfw.wmnet with OS bookworm</li> <li>17:36 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:35 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:34 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>17:32 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-worker1005.eqiad.wmnet with reason: host reimage</li> <li>17:31 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2157.codfw.wmnet with reason: host reimage</li> <li>17:30 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>17:29 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>17:27 jynus: rebuild frwiki.geo_tags @ an-redacteddb1001</li> <li>17:26 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2160.codfw.wmnet with reason: host reimage</li> <li>17:23 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2158.codfw.wmnet with reason: host reimage</li> <li>17:20 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2162.codfw.wmnet with reason: host reimage</li> <li>17:17 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2156.codfw.wmnet with reason: host reimage</li> <li>17:17 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:17 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:15 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker1005.eqiad.wmnet with OS bookworm</li> <li>17:14 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker1005.eqiad.wmnet - herron@cumin1002"</li> <li>17:14 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker1005.eqiad.wmnet - herron@cumin1002"</li> <li>17:14 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2161.codfw.wmnet with reason: host reimage</li> <li>17:14 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-worker1005.eqiad.wmnet on all recursors</li> <li>17:13 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-worker1005.eqiad.wmnet on all recursors</li> <li>17:13 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:13 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker1005.eqiad.wmnet - herron@cumin1002"</li> <li>17:13 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker1005.eqiad.wmnet - herron@cumin1002"</li> <li>17:11 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2159.codfw.wmnet with reason: host reimage</li> <li>17:10 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>17:09 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>17:09 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-worker1005.eqiad.wmnet</li> <li>17:08 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2158.codfw.wmnet with reason: host reimage</li> <li>17:08 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2144.codfw.wmnet with reason: host reimage</li> <li>17:08 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2145.codfw.wmnet with reason: host reimage</li> <li>17:08 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2157.codfw.wmnet with reason: host reimage</li> <li>17:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2161.codfw.wmnet with reason: host reimage</li> <li>17:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2160.codfw.wmnet with reason: host reimage</li> <li>17:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2162.codfw.wmnet with reason: host reimage</li> <li>17:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2156.codfw.wmnet with reason: host reimage</li> <li>17:07 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2159.codfw.wmnet with reason: host reimage</li> <li>17:07 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2163.codfw.wmnet with OS bookworm</li> <li>17:05 jhathaway@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>17:05 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>17:05 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>16:58 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host sretest2001.codfw.wmnet with OS bookworm</li> <li>16:58 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>16:55 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2162.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2161.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2160.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2159.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2158.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2157.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2156.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2145.codfw.wmnet with OS bookworm</li> <li>16:49 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2144.codfw.wmnet with OS bookworm</li> <li>16:43 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on sretest2001.codfw.wmnet with reason: host reimage</li> <li>16:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2136.codfw.wmnet with reason: host reimage</li> <li>16:35 elukey@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on sretest2001.codfw.wmnet with reason: host reimage</li> <li>16:35 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2136.codfw.wmnet with reason: host reimage</li> <li>16:25 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host sretest2001.codfw.wmnet with OS bookworm</li> <li>16:22 herron@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host aux-k8s-worker1004.eqiad.wmnet with OS bookworm</li> <li>16:16 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>16:10 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>16:05 herron@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on aux-k8s-worker1004.eqiad.wmnet with reason: host reimage</li> <li>16:02 herron@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on aux-k8s-worker1004.eqiad.wmnet with reason: host reimage</li> <li>16:02 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>15:55 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host sretest2001.codfw.wmnet with OS bookworm</li> <li>15:55 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host sretest2001.codfw.wmnet with OS bookworm</li> <li>15:48 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker1004.eqiad.wmnet with OS bookworm</li> <li>15:46 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2142.codfw.wmnet with OS bookworm</li> <li>15:46 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:45 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:45 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2143.codfw.wmnet with OS bookworm</li> <li>15:45 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:43 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:40 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2141.codfw.wmnet with OS bookworm</li> <li>15:40 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:39 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:32 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2129.codfw.wmnet with OS bookworm</li> <li>15:32 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:31 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>15:28 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2138.codfw.wmnet with OS bookworm</li> <li>15:28 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:28 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:27 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2137.codfw.wmnet with OS bookworm</li> <li>15:27 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2142.codfw.wmnet with reason: host reimage</li> <li>15:25 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>15:23 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2143.codfw.wmnet with reason: host reimage</li> <li>15:22 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:21 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>15:21 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:20 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2141.codfw.wmnet with reason: host reimage</li> <li>15:19 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host sretest2001.codfw.wmnet with OS bookworm</li> <li>15:18 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>15:16 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2087.codfw.wmnet with OS bullseye</li> <li>15:16 elukey@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>15:15 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2136.codfw.wmnet with reason: host reimage</li> <li>15:15 elukey@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>15:13 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2129.codfw.wmnet with reason: host reimage</li> <li>15:09 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2140.codfw.wmnet with reason: host reimage</li> <li>15:08 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host sretest2001.codfw.wmnet with OS bookworm</li> <li>15:06 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2138.codfw.wmnet with reason: host reimage</li> <li>15:05 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host sretest2001.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>15:03 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2137.codfw.wmnet with reason: host reimage</li> <li>15:01 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2142.codfw.wmnet with reason: host reimage</li> <li>15:01 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2143.codfw.wmnet with reason: host reimage</li> <li>15:01 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2141.codfw.wmnet with reason: host reimage</li> <li>15:00 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2140.codfw.wmnet with reason: host reimage</li> <li>15:00 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2128.codfw.wmnet with reason: host reimage</li> <li>14:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2138.codfw.wmnet with reason: host reimage</li> <li>14:57 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2136.codfw.wmnet with reason: host reimage</li> <li>14:56 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2137.codfw.wmnet with reason: host reimage</li> <li>14:56 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2129.codfw.wmnet with reason: host reimage</li> <li>14:56 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2128.codfw.wmnet with reason: host reimage</li> <li>14:56 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2087.codfw.wmnet with reason: host reimage</li> <li>14:55 elukey@cumin1002: START - Cookbook sre.hosts.provision for host sretest2001.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>14:52 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2087.codfw.wmnet with reason: host reimage</li> <li>14:42 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2143.codfw.wmnet with OS bookworm</li> <li>14:42 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2142.codfw.wmnet with OS bookworm</li> <li>14:42 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2141.codfw.wmnet with OS bookworm</li> <li>14:42 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2140.codfw.wmnet with OS bookworm</li> <li>14:42 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2139.codfw.wmnet with OS bookworm</li> <li>14:41 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2087.codfw.wmnet with OS bullseye</li> <li>14:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2138.codfw.wmnet with OS bookworm</li> <li>14:38 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2137.codfw.wmnet with OS bookworm</li> <li>14:38 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>14:38 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2136.codfw.wmnet with OS bookworm</li> <li>14:38 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2129.codfw.wmnet with OS bookworm</li> <li>14:38 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2128.codfw.wmnet with OS bookworm</li> <li>14:37 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>14:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2128']</li> <li>14:34 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2128']</li> <li>14:34 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2158']</li> <li>14:34 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2158']</li> <li>14:34 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2157']</li> <li>14:34 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2157']</li> <li>14:34 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2156']</li> <li>14:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2156']</li> <li>14:33 jhancock@cumin2002: END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['wikikube-worker2156']</li> <li>14:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2156']</li> <li>14:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2145']</li> <li>14:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2145']</li> <li>14:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2144']</li> <li>14:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2144']</li> <li>14:33 jhancock@cumin2002: END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['wikikube-worker2144']</li> <li>14:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2144']</li> <li>14:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2143']</li> <li>14:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2143']</li> <li>14:32 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2142']</li> <li>14:31 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2142']</li> <li>14:31 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2141']</li> <li>14:30 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2141']</li> <li>14:30 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2140']</li> <li>14:30 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2140']</li> <li>14:29 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2139']</li> <li>14:29 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2139']</li> <li>14:29 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2138']</li> <li>14:29 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2138']</li> <li>14:29 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2137']</li> <li>14:29 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2137']</li> <li>14:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2136']</li> <li>14:28 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2136']</li> <li>14:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2129']</li> <li>14:28 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2129']</li> <li>14:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2128']</li> <li>14:27 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2128']</li> <li>14:18 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>14:18 elukey@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>13:31 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>13:30 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:29 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>12:32 hnowlan@deploy1003: helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>12:30 hnowlan@deploy1003: helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>12:30 hnowlan@deploy1003: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>12:30 hnowlan@deploy1003: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>12:29 hnowlan@deploy1003: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>12:28 hnowlan@deploy1003: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>12:07 elukey@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>12:04 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2087.codfw.wmnet with OS bullseye</li> <li>11:59 apergos: testing of account creation backfill script on mwmaint2001 complete for the moment</li> <li>11:53 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2087.codfw.wmnet with OS bullseye</li> <li>11:51 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2086.codfw.wmnet with reason: host reimage</li> <li>11:48 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2086.codfw.wmnet with reason: host reimage</li> <li>11:37 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2087.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:37 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>11:27 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2087.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:25 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti2016.codfw.wmnet</li> <li>11:25 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>11:25 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2016.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>11:24 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2016.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>11:17 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>11:16 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>11:13 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2086.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:13 elukey@cumin2002: START - Cookbook sre.hosts.provision for host ms-be2086.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>11:13 elukey@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>11:07 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>11:05 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>11:04 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>11:00 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:58 elukey@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:56 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti2016.codfw.wmnet</li> <li>10:56 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti2015.codfw.wmnet</li> <li>10:56 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>10:56 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2015.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>10:55 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2015.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>10:51 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>10:45 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti2015.codfw.wmnet</li> <li>10:45 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:39 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>10:34 elukey@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:29 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:19 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1011.eqiad.wmnet</li> <li>10:18 elukey@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:16 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2086.codfw.wmnet with OS bullseye</li> <li>10:16 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1011.eqiad.wmnet</li> <li>10:02 gmodena@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mw-dump-rev-content-reconcile-enrich: apply</li> <li>10:01 gmodena@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mw-dump-rev-content-reconcile-enrich: apply</li> <li>09:57 apergos: testing account creation backfill script on mwmaint2001 in screen session as ariel</li> <li>09:49 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2086.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:41 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2085.codfw.wmnet with OS bullseye</li> <li>09:41 elukey@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin2002"</li> <li>09:39 elukey@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin2002"</li> <li>09:38 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2086.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>09:29 stevemunene@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10 days, 0:00:00 on an-presto1018.eqiad.wmnet with reason: Downtimed for further troubleshooting possible Hardware failure</li> <li>09:29 stevemunene@cumin1002: START - Cookbook sre.hosts.downtime for 10 days, 0:00:00 on an-presto1018.eqiad.wmnet with reason: Downtimed for further troubleshooting possible Hardware failure</li> <li>09:24 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2085.codfw.wmnet with reason: host reimage</li> <li>09:20 elukey@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2085.codfw.wmnet with reason: host reimage</li> <li>09:09 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2085.codfw.wmnet with OS bullseye</li> <li>09:09 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2085.codfw.wmnet with OS bullseye</li> <li>09:03 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device ssw1-a8-codfw</li> <li>09:03 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device ssw1-a8-codfw</li> <li>09:03 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device ssw1-a1-codfw</li> <li>09:03 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device ssw1-a1-codfw</li> <li>09:01 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b8-codfw</li> <li>09:01 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b8-codfw</li> <li>09:01 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b7-codfw</li> <li>09:01 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b7-codfw</li> <li>08:56 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2085.codfw.wmnet with OS bullseye</li> <li>08:54 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b6-codfw</li> <li>08:54 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b6-codfw</li> <li>08:53 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b5-codfw</li> <li>08:53 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b5-codfw</li> <li>08:53 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b4-codfw</li> <li>08:52 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b4-codfw</li> <li>08:52 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b3-codfw</li> <li>08:52 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b3-codfw</li> <li>08:52 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-b2-codfw</li> <li>08:52 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-b2-codfw</li> <li>08:44 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a8-codfw</li> <li>08:43 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a8-codfw</li> <li>08:43 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a7-codfw</li> <li>08:43 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a7-codfw</li> <li>08:43 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1048.eqiad.wmnet to cluster eqiad and group C</li> <li>08:43 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a6-codfw</li> <li>08:43 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a6-codfw</li> <li>08:42 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a5-codfw</li> <li>08:42 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a5-codfw</li> <li>08:42 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1048.eqiad.wmnet to cluster eqiad and group C</li> <li>08:42 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a4-codfw</li> <li>08:41 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a4-codfw</li> <li>08:41 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a3-codfw</li> <li>08:41 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a3-codfw</li> <li>08:41 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2085.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:41 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-a2-codfw</li> <li>08:40 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device lsw1-a2-codfw</li> <li>08:39 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device ssw1-f1-eqiad</li> <li>08:39 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device ssw1-f1-eqiad</li> <li>08:35 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device ssw1-e1-eqiad</li> <li>08:35 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device ssw1-e1-eqiad</li> <li>08:34 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw2-d5-eqiad</li> <li>08:34 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>08:34 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device cloudsw2-d5-eqiad</li> <li>08:33 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>08:31 elukey@cumin2002: START - Cookbook sre.hosts.provision for host ms-be2085.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:30 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cr2-eqsin</li> <li>08:30 ayounsi@cumin1002: START - Cookbook sre.network.tls for network device cr2-eqsin</li> <li>08:27 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>08:27 elukey@cumin2002: START - Cookbook sre.hosts.provision for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>08:26 moritzm: upgraded ircstream on irc.wikimedia.org to 1.0.1</li> <li>08:08 XioNoX: update gnmic to 0.39 on all netflow hosts</li> <li>08:05 XioNoX: add gnmic 0.39 from official git repo to bookworm reprepro - <a href="https://phabricator.wikimedia.org/T347461" class="extiw" title="phab:T347461">T347461</a></li> <li>07:48 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1047.eqiad.wmnet to cluster eqiad and group C</li> <li>07:48 XioNoX: manually install/test gnmic 0.39 on netflow6001</li> <li>07:46 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1047.eqiad.wmnet to cluster eqiad and group C</li> <li>07:45 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1048.eqiad.wmnet</li> <li>07:39 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1048.eqiad.wmnet</li> <li>07:39 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1047.eqiad.wmnet</li> <li>07:33 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1047.eqiad.wmnet</li> <li>07:33 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1047.eqiad.wmnet to cluster eqiad and group C</li> <li>07:33 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1047.eqiad.wmnet to cluster eqiad and group C</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-07" data-mw-thread-id="h-2024-11-07"><span data-mw-comment-start="" id="h-2024-11-07"></span>2024-11-07<span data-mw-comment-end="h-2024-11-07"></span></h2></div> <ul><li>23:00 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>22:48 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2170.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:47 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2169.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:47 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2168.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:46 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2167.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:45 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2166.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:44 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2165.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:43 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2164.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2163.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:41 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2162.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:41 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2161.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:40 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2160.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2141.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2159.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2158.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:37 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2157.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:37 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2170.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:37 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:37 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2156.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:37 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2169.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:36 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2168.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2145.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:35 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2167.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:34 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2144.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:34 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2166.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:34 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>22:34 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2143.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2142.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:33 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2165.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:32 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2164.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2163.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2162.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2140.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2139.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2161.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:29 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2160.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:28 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2159.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2138.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2137.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:27 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2158.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2136.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:27 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2157.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:26 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2129.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2156.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2145.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2128.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2144.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:23 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2143.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:22 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2142.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:22 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>22:21 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2141.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:20 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2140.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:19 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>22:19 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2139.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:17 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2138.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:17 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2137.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:16 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2136.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:15 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2129.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:14 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2128.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:12 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wdqs2026.codfw.wmnet with OS bullseye</li> <li>22:12 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>22:10 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>22:08 jhathaway@cumin2002: START - Cookbook sre.hosts.provision for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>22:07 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wdqs2027.codfw.wmnet with OS bullseye</li> <li>22:07 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>22:06 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:58 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:58 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2170 to codfw - jhancock@cumin2002"</li> <li>21:58 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2170 to codfw - jhancock@cumin2002"</li> <li>21:53 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>21:53 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wdqs2026.codfw.wmnet with reason: host reimage</li> <li>21:52 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:51 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2166 to codfw - jhancock@cumin2002"</li> <li>21:50 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2166 to codfw - jhancock@cumin2002"</li> <li>21:50 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wdqs2027.codfw.wmnet with reason: host reimage</li> <li>21:47 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>21:46 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wdqs2026.codfw.wmnet with reason: host reimage</li> <li>21:46 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wdqs2027.codfw.wmnet with reason: host reimage</li> <li>21:41 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:34 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:34 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2158 to codfw - jhancock@cumin2002"</li> <li>21:33 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2158 to codfw - jhancock@cumin2002"</li> <li>21:30 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>21:27 jhathaway@cumin2002: START - Cookbook sre.hosts.provision for host ms-be2082.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:26 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:26 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2143 to codfw - jhancock@cumin2002"</li> <li>21:26 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2143 to codfw - jhancock@cumin2002"</li> <li>21:22 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>21:21 jhathaway@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>21:18 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wdqs2027.codfw.wmnet with OS bullseye</li> <li>21:18 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wdqs2026.codfw.wmnet with OS bullseye</li> <li>21:18 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wdqs2027']</li> <li>21:17 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wdqs2026']</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wdqs2027']</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wdqs2026']</li> <li>21:11 herron@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host aux-k8s-worker1004.eqiad.wmnet with OS bookworm</li> <li>21:11 jsn@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1084883" class="extiw" title="gerrit:1084883">Enable AutoModerator on viwiki (T378343)</a> (duration: 08m 28s)</li> <li>21:09 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker1004.eqiad.wmnet with OS bookworm</li> <li>21:06 jsn@deploy2002: suecarmol, jsn: Continuing with sync</li> <li>21:06 jsn@deploy2002: suecarmol, jsn: Backport for <a href="https://gerrit.wikimedia.org/r/1084883" class="extiw" title="gerrit:1084883">Enable AutoModerator on viwiki (T378343)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:03 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:03 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2128 to codfw - jhancock@cumin2002"</li> <li>21:03 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2128 to codfw - jhancock@cumin2002"</li> <li>21:03 jhathaway@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>21:02 jsn@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1084883" class="extiw" title="gerrit:1084883">Enable AutoModerator on viwiki (T378343)</a></li> <li>21:01 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wdqs2027.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:01 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wdqs2026.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:59 jhathaway@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>20:59 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>20:50 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wdqs2027.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:50 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wdqs2026.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:49 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:49 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wdqs2026 to codfw - jhancock@cumin2002"</li> <li>20:49 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wdqs2026 to codfw - jhancock@cumin2002"</li> <li>20:46 jhathaway@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bookworm</li> <li>20:43 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>20:35 cdanis@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087987" class="extiw" title="gerrit:1087987">Enable Chart extension on testwiki and testcommonswiki (T378127)</a> (duration: 13m 02s)</li> <li>20:30 cdanis@deploy2002: cdanis, aude: Continuing with sync</li> <li>20:25 cdanis@deploy2002: cdanis, aude: Backport for <a href="https://gerrit.wikimedia.org/r/1087987" class="extiw" title="gerrit:1087987">Enable Chart extension on testwiki and testcommonswiki (T378127)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>20:22 cdanis@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087987" class="extiw" title="gerrit:1087987">Enable Chart extension on testwiki and testcommonswiki (T378127)</a></li> <li>20:21 cdanis@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087975" class="extiw" title="gerrit:1087975">DB config for testcommonswiki deployment for Charts (T379199)</a> (duration: 10m 45s)</li> <li>20:15 cdanis@deploy2002: cdanis, bvibber: Continuing with sync</li> <li>20:13 cdanis@deploy2002: cdanis, bvibber: Backport for <a href="https://gerrit.wikimedia.org/r/1087975" class="extiw" title="gerrit:1087975">DB config for testcommonswiki deployment for Charts (T379199)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>20:10 cdanis@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087975" class="extiw" title="gerrit:1087975">DB config for testcommonswiki deployment for Charts (T379199)</a></li> <li>20:02 dduvall@deploy2002: Installing scap version "4.122.0" for 209 hosts</li> <li>19:42 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>19:42 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: add dummy record for pfw1-eqiad.wikimedia.org - cmooney@cumin1002"</li> <li>19:42 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: add dummy record for pfw1-eqiad.wikimedia.org - cmooney@cumin1002"</li> <li>19:37 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>19:33 cmooney@cumin1002: END (ERROR) - Cookbook sre.dns.netbox (exit_code=97)</li> <li>19:33 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>19:23 cdanis: <a href="https://phabricator.wikimedia.org/T379199" class="extiw" title="phab:T379199">T379199</a> 💙cdanis@mwmaint2002.codfw.wmnet ~ 🕝☕ mwscript sql.php --wiki=testcommonswiki /srv/mediawiki/php-1.44.0-wmf.2/extensions/JsonConfig/sql/mysql/tables-generated.sql</li> <li>19:19 dzahn@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on vrts1003.eqiad.wmnet with reason: nftables</li> <li>19:19 dzahn@cumin2002: START - Cookbook sre.hosts.downtime for 0:10:00 on vrts1003.eqiad.wmnet with reason: nftables</li> <li>19:18 aokoth@cumin1002: END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host vrts1003.eqiad.wmnet</li> <li>19:11 dzahn@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on vrts1003.eqiad.wmnet with reason: nftables</li> <li>19:11 dzahn@cumin2002: START - Cookbook sre.hosts.downtime for 0:10:00 on vrts1003.eqiad.wmnet with reason: nftables</li> <li>19:10 dzahn@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on vrts2002.codfw.wmnet with reason: nftables</li> <li>19:10 dzahn@cumin2002: START - Cookbook sre.hosts.downtime for 0:10:00 on vrts2002.codfw.wmnet with reason: nftables</li> <li>19:08 mutante: VRTS - switching firewall provider from iptables to nftables</li> <li>19:06 aokoth@cumin1002: START - Cookbook sre.hosts.reboot-single for host vrts1003.eqiad.wmnet</li> <li>19:03 herron@cumin1002: END (FAIL) - Cookbook sre.ganeti.makevm (exit_code=99) for new host aux-k8s-worker1004.eqiad.wmnet</li> <li>19:03 herron@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host aux-k8s-worker1004.eqiad.wmnet with OS bookworm</li> <li>19:00 herron@cumin1002: START - Cookbook sre.hosts.reimage for host aux-k8s-worker1004.eqiad.wmnet with OS bookworm</li> <li>18:59 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker1004.eqiad.wmnet - herron@cumin1002"</li> <li>18:59 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM aux-k8s-worker1004.eqiad.wmnet - herron@cumin1002"</li> <li>18:59 herron@cumin1002: END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) aux-k8s-worker1004.eqiad.wmnet on all recursors</li> <li>18:59 herron@cumin1002: START - Cookbook sre.dns.wipe-cache aux-k8s-worker1004.eqiad.wmnet on all recursors</li> <li>18:59 herron@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>18:58 herron@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker1004.eqiad.wmnet - herron@cumin1002"</li> <li>18:58 herron@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM aux-k8s-worker1004.eqiad.wmnet - herron@cumin1002"</li> <li>18:50 herron@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:50 herron@cumin1002: START - Cookbook sre.ganeti.makevm for new host aux-k8s-worker1004.eqiad.wmnet</li> <li>18:43 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>18:43 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2138 to codfw - jhancock@cumin2002"</li> <li>18:43 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2138 to codfw - jhancock@cumin2002"</li> <li>18:14 swfrench-wmf: updated changeprop-jobqueue to 2024-11-05-170900-production - <a href="https://phabricator.wikimedia.org/T356241" class="extiw" title="phab:T356241">T356241</a></li> <li>18:13 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>18:11 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>18:01 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:59 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:58 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:57 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply</li> <li>17:55 fnegri@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for cloudvirt1063.eqiad.wmnet</li> <li>17:55 fnegri@cumin1002: START - Cookbook sre.hosts.remove-downtime for cloudvirt1063.eqiad.wmnet</li> <li>17:48 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/changeprop: apply</li> <li>17:48 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/changeprop: apply</li> <li>17:44 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/changeprop: apply</li> <li>17:43 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/changeprop: apply</li> <li>17:42 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/changeprop: apply</li> <li>17:41 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/changeprop: apply</li> <li>17:29 fnegri@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudvirt1063.eqiad.wmnet with OS bookworm</li> <li>17:29 fnegri@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - fnegri@cumin1002"</li> <li>17:27 fnegri@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - fnegri@cumin1002"</li> <li>17:18 cmooney@cumin1002: END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device fasw2-c1a-eqiad</li> <li>17:16 cmooney@cumin1002: START - Cookbook sre.network.tls for network device fasw2-c1a-eqiad</li> <li>17:12 rzl: manually run mediawiki_job_wikimediaevents-UpdatePeriodicMetrics-global # <a href="https://phabricator.wikimedia.org/T375508" class="extiw" title="phab:T375508">T375508</a></li> <li>17:09 arlolra@deploy2002: helmfile [codfw] DONE helmfile.d/services/mobileapps: apply</li> <li>17:08 arlolra@deploy2002: helmfile [codfw] START helmfile.d/services/mobileapps: apply</li> <li>17:06 rzl: manually run mediawiki_job_wikimediaevents-UpdatePeriodicMetrics-per-wiki # <a href="https://phabricator.wikimedia.org/T375508" class="extiw" title="phab:T375508">T375508</a></li> <li>17:03 arlolra@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply</li> <li>17:02 arlolra@deploy2002: helmfile [eqiad] START helmfile.d/services/mobileapps: apply</li> <li>17:01 fnegri@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1063.eqiad.wmnet with reason: host reimage</li> <li>16:57 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>16:57 elukey@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin2002"</li> <li>16:57 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2084.codfw.wmnet with OS bullseye</li> <li>16:57 arlolra@deploy2002: helmfile [codfw] DONE helmfile.d/services/mobileapps: apply</li> <li>16:56 arlolra@deploy2002: helmfile [codfw] START helmfile.d/services/mobileapps: apply</li> <li>16:56 arlolra@deploy2002: helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply</li> <li>16:56 fnegri@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1063.eqiad.wmnet with reason: host reimage</li> <li>16:54 arlolra@deploy2002: helmfile [eqiad] START helmfile.d/services/mobileapps: apply</li> <li>16:54 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>16:48 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:48 elukey@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:46 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2084.codfw.wmnet with OS bullseye</li> <li>16:45 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2084.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>16:41 fnegri@cumin1002: START - Cookbook sre.hosts.reimage for host cloudvirt1063.eqiad.wmnet with OS bookworm</li> <li>16:34 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2084.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>16:32 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2083.codfw.wmnet with reason: host reimage</li> <li>16:28 elukey@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin2002"</li> <li>16:28 elukey@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2083.codfw.wmnet with reason: host reimage</li> <li>16:24 arlolra@deploy2002: helmfile [staging] DONE helmfile.d/services/mobileapps: apply</li> <li>16:23 arlolra@deploy2002: helmfile [staging] START helmfile.d/services/mobileapps: apply</li> <li>16:15 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>16:07 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>16:04 elukey@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2082.codfw.wmnet with reason: host reimage</li> <li>15:57 herron@cumin1002: END (PASS) - Cookbook sre.kafka.roll-restart-reboot-brokers (exit_code=0) rolling restart_daemons on A:kafka-logging-eqiad</li> <li>15:54 moritzm: remove ganeti1010 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>15:53 joelyrookewmde: Finished populateSitesTable for tcywiktionary (<a href="https://phabricator.wikimedia.org/T378466" class="extiw" title="phab:T378466">T378466</a>) and tcywikisource (<a href="https://phabricator.wikimedia.org/T378474" class="extiw" title="phab:T378474">T378474</a>)</li> <li>15:53 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>15:52 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1010.eqiad.wmnet</li> <li>15:39 jgiannelos@deploy2002: Finished deploy [restbase/deploy@6d0b97e]: Add new wikis to RESTBase (duration: 21m 33s)</li> <li>15:33 herron@cumin1002: START - Cookbook sre.kafka.roll-restart-reboot-brokers rolling restart_daemons on A:kafka-logging-eqiad</li> <li>15:31 taavi: taavi@deploy2002 ~ $ mwscript-k8s migrateUserGroup.php -- --wiki=labswiki contentadmin sysop # <a href="https://phabricator.wikimedia.org/T375950" class="extiw" title="phab:T375950">T375950</a></li> <li>15:31 joelyrookewmde: joelyrookewmde@mwmaint2002:~$ foreachwikiindblist wikidataclient extensions/Wikibase/lib/maintenance/populateSitesTable.php --force-protocol https</li> <li>15:29 herron@cumin1002: END (PASS) - Cookbook sre.kafka.roll-restart-reboot-brokers (exit_code=0) rolling restart_daemons on A:kafka-logging-codfw</li> <li>15:18 jgiannelos@deploy2002: Started deploy [restbase/deploy@6d0b97e]: Add new wikis to RESTBase</li> <li>15:16 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>15:15 jnuche@deploy2002: Finished deploy [releng/jenkins-deploy@abc27c0] (releasing): (no justification provided) (duration: 01m 13s)</li> <li>15:14 jnuche@deploy2002: Started deploy [releng/jenkins-deploy@abc27c0] (releasing): (no justification provided)</li> <li>15:11 jnuche@deploy2002: Finished deploy [releng/jenkins-deploy@abc27c0] (releasing): (no justification provided) (duration: 00m 52s)</li> <li>15:10 jnuche@deploy2002: Started deploy [releng/jenkins-deploy@abc27c0] (releasing): (no justification provided)</li> <li>15:07 herron@cumin1002: START - Cookbook sre.kafka.roll-restart-reboot-brokers rolling restart_daemons on A:kafka-logging-codfw</li> <li>14:55 hashar: Restarted CI Jenkins for plugins update</li> <li>14:41 moritzm: installing python-git security updates</li> <li>14:29 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2082.codfw.wmnet with OS bullseye</li> <li>14:25 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087927" class="extiw" title="gerrit:1087927">Deploy EditCheck (references) to hiwiki, bnwiki, idwiki (T366381)</a> (duration: 09m 37s)</li> <li>14:20 lucaswerkmeister-wmde@deploy2002: esanders, lucaswerkmeister-wmde: Continuing with sync</li> <li>14:18 lucaswerkmeister-wmde@deploy2002: esanders, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1087927" class="extiw" title="gerrit:1087927">Deploy EditCheck (references) to hiwiki, bnwiki, idwiki (T366381)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:15 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>14:15 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087927" class="extiw" title="gerrit:1087927">Deploy EditCheck (references) to hiwiki, bnwiki, idwiki (T366381)</a></li> <li>14:13 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088215" class="extiw" title="gerrit:1088215">Enable Section Translation in ann, iba, nr and, tdd Wikipedias (T371420)</a> (duration: 10m 08s)</li> <li>14:09 kartik@deploy2002: kartik: Continuing with sync</li> <li>14:06 kartik@deploy2002: kartik: Backport for <a href="https://gerrit.wikimedia.org/r/1088215" class="extiw" title="gerrit:1088215">Enable Section Translation in ann, iba, nr and, tdd Wikipedias (T371420)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:04 joal@deploy2002: Finished deploy [airflow-dags/analytics@23bc4ad]: Regular analytics weekly train [airflow-dags/analytics@23bc4ad3] (duration: 01m 44s)</li> <li>14:03 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1088215" class="extiw" title="gerrit:1088215">Enable Section Translation in ann, iba, nr and, tdd Wikipedias (T371420)</a></li> <li>14:03 joal@deploy2002: Started deploy [airflow-dags/analytics@23bc4ad]: Regular analytics weekly train [airflow-dags/analytics@23bc4ad3]</li> <li>13:52 cwhite: running thanos bucket cleanup on titan1001 - <a href="https://phabricator.wikimedia.org/T351927" class="extiw" title="phab:T351927">T351927</a></li> <li>13:37 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1048</li> <li>13:36 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1048</li> <li>13:35 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1047</li> <li>13:34 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1047</li> <li>13:23 joal@deploy2002: Finished deploy [analytics/refinery@4bec064] (hadoop-test): Regular analytics weekly train TEST [analytics/refinery@4bec0640] (duration: 03m 44s)</li> <li>13:20 joal@deploy2002: Started deploy [analytics/refinery@4bec064] (hadoop-test): Regular analytics weekly train TEST [analytics/refinery@4bec0640]</li> <li>13:13 joal@deploy2002: Finished deploy [analytics/refinery@4bec064] (thin): Regular analytics weekly train THIN [analytics/refinery@4bec0640] (duration: 05m 03s)</li> <li>13:08 joal@deploy2002: Started deploy [analytics/refinery@4bec064] (thin): Regular analytics weekly train THIN [analytics/refinery@4bec0640]</li> <li>12:53 joal@deploy2002: Finished deploy [analytics/refinery@4bec064]: Regular analytics weekly train [analytics/refinery@4bec0640] (duration: 16m 47s)</li> <li>12:40 jmm@cumin2002: END (FAIL) - Cookbook sre.network.configure-switch-interfaces (exit_code=99) for host ganeti1047</li> <li>12:40 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1047</li> <li>12:39 jmm@cumin2002: END (FAIL) - Cookbook sre.network.configure-switch-interfaces (exit_code=99) for host ganeti1047</li> <li>12:37 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1047</li> <li>12:36 joal@deploy2002: Started deploy [analytics/refinery@4bec064]: Regular analytics weekly train [analytics/refinery@4bec0640]</li> <li>12:16 vgutierrez: repool liberica on lvs1013</li> <li>11:44 sfaci@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mpic: apply</li> <li>11:44 sfaci@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mpic: apply</li> <li>11:27 jgiannelos@deploy2002: helmfile [eqiad] DONE helmfile.d/services/proton: sync</li> <li>11:26 jgiannelos@deploy2002: helmfile [eqiad] START helmfile.d/services/proton: sync</li> <li>11:26 jgiannelos@deploy2002: helmfile [codfw] DONE helmfile.d/services/proton: sync</li> <li>11:25 jgiannelos@deploy2002: helmfile [codfw] START helmfile.d/services/proton: sync</li> <li>11:24 jgiannelos@deploy2002: helmfile [staging] DONE helmfile.d/services/proton: sync</li> <li>11:24 jgiannelos@deploy2002: helmfile [staging] START helmfile.d/services/proton: sync</li> <li>11:19 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-editquality-reverted' for release 'main' .</li> <li>11:19 sfaci@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mpic-next: apply</li> <li>11:19 sfaci@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mpic-next: apply</li> <li>11:18 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-editquality-goodfaith' for release 'main' .</li> <li>11:17 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-editquality-damaging' for release 'main' .</li> <li>11:17 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-drafttopic' for release 'main' .</li> <li>11:17 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-draftquality' for release 'main' .</li> <li>11:17 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-articletopic' for release 'main' .</li> <li>11:16 isaranto@deploy2002: helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-articlequality' for release 'main' .</li> <li>11:11 isaranto@deploy2002: helmfile [ml-serve-codfw] Ran 'sync' command on namespace 'revscoring-editquality-reverted' for release 'main' .</li> <li>11:10 isaranto@deploy2002: helmfile [ml-serve-codfw] Ran 'sync' command on namespace 'revscoring-editquality-goodfaith' for release 'main' .</li> <li>11:09 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1010.eqiad.wmnet</li> <li>11:09 isaranto@deploy2002: helmfile [ml-serve-codfw] Ran 'sync' command on namespace 'revscoring-editquality-damaging' for release 'main' .</li> <li>11:04 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1010.eqiad.wmnet</li> <li>11:03 vgutierrez: depool liberica on lvs1013</li> <li>11:01 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1010.eqiad.wmnet</li> <li>10:58 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2082.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:55 jmm@cumin2002: END (PASS) - Cookbook sre.kafka.roll-restart-reboot-brokers (exit_code=0) rolling restart_daemons on A:kafka-test-eqiad</li> <li>10:48 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2082.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>10:41 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2081.codfw.wmnet with OS bullseye</li> <li>10:41 elukey@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin2002"</li> <li>10:40 elukey@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin2002"</li> <li>10:40 gmodena@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mw-dump-rev-content-reconcile-enrich: apply</li> <li>10:40 gmodena@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mw-dump-rev-content-reconcile-enrich: apply</li> <li>10:33 jmm@cumin2002: START - Cookbook sre.kafka.roll-restart-reboot-brokers rolling restart_daemons on A:kafka-test-eqiad</li> <li>10:21 elukey@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2081.codfw.wmnet with reason: host reimage</li> <li>10:20 gmodena@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mw-dump-rev-content-reconcile-enrich: apply</li> <li>10:20 gmodena@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mw-dump-rev-content-reconcile-enrich: apply</li> <li>10:18 elukey@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2081.codfw.wmnet with reason: host reimage</li> <li>10:07 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2081.codfw.wmnet with OS bullseye</li> <li>10:02 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1009.eqiad.wmnet</li> <li>09:58 oblivian@cumin2002: END (PASS) - Cookbook sre.deploy.hiddenparma (exit_code=0) Hiddenparma deployment to the alerting hosts with reason: "Add rw interface (still disabled), search - oblivian@cumin2002"</li> <li>09:58 oblivian@cumin2002: END (PASS) - Cookbook sre.deploy.python-code (exit_code=0) hiddenparma to alert[1002,2002].wikimedia.org with reason: Add rw interface (still disabled), search - oblivian@cumin2002</li> <li>09:57 oblivian@cumin2002: START - Cookbook sre.deploy.python-code hiddenparma to alert[1002,2002].wikimedia.org with reason: Add rw interface (still disabled), search - oblivian@cumin2002</li> <li>09:57 oblivian@cumin2002: START - Cookbook sre.deploy.hiddenparma Hiddenparma deployment to the alerting hosts with reason: "Add rw interface (still disabled), search - oblivian@cumin2002"</li> <li>09:52 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70981">https://phabricator.wikimedia.org/P70981</a> and previous config saved to /var/cache/conftool/dbconfig/20241107-095205-arnaudb.json</li> <li>09:51 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1009.eqiad.wmnet</li> <li>09:41 elukey@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2081.codfw.wmnet with OS bullseye</li> <li>09:36 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70980">https://phabricator.wikimedia.org/P70980</a> and previous config saved to /var/cache/conftool/dbconfig/20241107-093657-arnaudb.json</li> <li>09:29 vgutierrez: upload liberica 0.4 to apt.wm.o (bookworm-wikimedia)</li> <li>09:21 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70979">https://phabricator.wikimedia.org/P70979</a> and previous config saved to /var/cache/conftool/dbconfig/20241107-092150-arnaudb.json</li> <li>09:21 moritzm: installing openjdk-8 security updates</li> <li>09:21 moritzm: uploaded openjdk-8 8u412-ga-1~deb11u1 to apt.wikimedia.org for bookworm-wikimedia</li> <li>09:14 jnuche@deploy2002: rebuilt and synchronized wikiversions files: group2 to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>09:06 arnaudb@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70978">https://phabricator.wikimedia.org/P70978</a> and previous config saved to /var/cache/conftool/dbconfig/20241107-090643-arnaudb.json</li> <li>08:41 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2081.codfw.wmnet with OS bullseye</li> <li>08:40 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2081.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:27 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2081.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:26 kartik@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087914" class="extiw" title="gerrit:1087914">Translate: Enable message bundle Scribunto module on testwiki (T359918)</a> (duration: 18m 39s)</li> <li>08:25 _joe_: runing scap pull on mwdebug2001/2002</li> <li>08:19 kartik@deploy2002: kartik, abi: Continuing with sync</li> <li>08:13 kartik@deploy2002: kartik, abi: Backport for <a href="https://gerrit.wikimedia.org/r/1087914" class="extiw" title="gerrit:1087914">Translate: Enable message bundle Scribunto module on testwiki (T359918)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>08:07 kartik@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087914" class="extiw" title="gerrit:1087914">Translate: Enable message bundle Scribunto module on testwiki (T359918)</a></li> <li>08:06 arnaudb@cumin1002: dbctl commit (dc=all): 'Depooling db2155 (<a href="https://phabricator.wikimedia.org/T367781" class="extiw" title="phab:T367781">T367781</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70977">https://phabricator.wikimedia.org/P70977</a> and previous config saved to /var/cache/conftool/dbconfig/20241107-080618-arnaudb.json</li> <li>08:06 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2187.codfw.wmnet with reason: Maintenance</li> <li>08:05 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 8:00:00 on db2187.codfw.wmnet with reason: Maintenance</li> <li>08:05 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2155.codfw.wmnet with reason: Maintenance</li> <li>08:05 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 4:00:00 on db2155.codfw.wmnet with reason: Maintenance</li> <li>07:50 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:50 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 5 days, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:50 arnaudb@cumin1002: END (ERROR) - Cookbook sre.hosts.downtime (exit_code=97) for 1 day, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:50 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>07:28 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1046.eqiad.wmnet to cluster eqiad and group C</li> <li>07:27 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1046.eqiad.wmnet to cluster eqiad and group C</li> <li>07:27 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1045.eqiad.wmnet to cluster eqiad and group C</li> <li>07:25 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1045.eqiad.wmnet to cluster eqiad and group C</li> <li>07:25 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1045.eqiad.wmnet to cluster eqiad and group B</li> <li>07:25 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1045.eqiad.wmnet to cluster eqiad and group B</li> <li>07:18 kartik@deploy2002: helmfile [eqiad] DONE helmfile.d/services/machinetranslation: apply</li> <li>07:03 kartik@deploy2002: helmfile [eqiad] START helmfile.d/services/machinetranslation: apply</li> <li>06:55 kartik@deploy2002: helmfile [codfw] DONE helmfile.d/services/machinetranslation: apply</li> <li>06:47 kartik@deploy2002: helmfile [codfw] START helmfile.d/services/machinetranslation: apply</li> <li>06:44 kartik@deploy2002: helmfile [staging] DONE helmfile.d/services/machinetranslation: apply</li> <li>06:39 kartik@deploy2002: helmfile [staging] START helmfile.d/services/machinetranslation: apply</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-06" data-mw-thread-id="h-2024-11-06"><span data-mw-comment-start="" id="h-2024-11-06"></span>2024-11-06<span data-mw-comment-end="h-2024-11-06"></span></h2></div> <ul><li>23:46 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2152.codfw.wmnet with OS bookworm</li> <li>23:46 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:45 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:41 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc-gp1006.eqiad.wmnet with OS bookworm</li> <li>23:41 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>23:41 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>23:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2151.codfw.wmnet with OS bookworm</li> <li>23:39 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:37 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2154.codfw.wmnet with OS bookworm</li> <li>23:36 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:34 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:31 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc-gp1005.eqiad.wmnet with OS bookworm</li> <li>23:31 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>23:30 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>23:28 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2153.codfw.wmnet with OS bookworm</li> <li>23:28 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:28 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2152.codfw.wmnet with reason: host reimage</li> <li>23:23 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc-gp1004.eqiad.wmnet with OS bookworm</li> <li>23:23 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>23:23 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jclark@cumin1002"</li> <li>23:23 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2155.codfw.wmnet with OS bookworm</li> <li>23:23 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:22 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc-gp1006.eqiad.wmnet with reason: host reimage</li> <li>23:19 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2151.codfw.wmnet with reason: host reimage</li> <li>23:18 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:15 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2154.codfw.wmnet with reason: host reimage</li> <li>23:12 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc-gp1005.eqiad.wmnet with reason: host reimage</li> <li>23:08 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2153.codfw.wmnet with reason: host reimage</li> <li>23:05 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc-gp1004.eqiad.wmnet with reason: host reimage</li> <li>23:02 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on mc-gp1005.eqiad.wmnet with reason: host reimage</li> <li>23:02 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2155.codfw.wmnet with reason: host reimage</li> <li>23:00 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on mc-gp1004.eqiad.wmnet with reason: host reimage</li> <li>23:00 jclark@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on mc-gp1006.eqiad.wmnet with reason: host reimage</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2153.codfw.wmnet with reason: host reimage</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2152.codfw.wmnet with reason: host reimage</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2151.codfw.wmnet with reason: host reimage</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2154.codfw.wmnet with reason: host reimage</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2155.codfw.wmnet with reason: host reimage</li> <li>22:44 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host mc-gp1004.eqiad.wmnet with OS bookworm</li> <li>22:44 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host mc-gp1005.eqiad.wmnet with OS bookworm</li> <li>22:43 jclark@cumin1002: START - Cookbook sre.hosts.reimage for host mc-gp1006.eqiad.wmnet with OS bookworm</li> <li>22:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2155.codfw.wmnet with OS bookworm</li> <li>22:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2154.codfw.wmnet with OS bookworm</li> <li>22:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2153.codfw.wmnet with OS bookworm</li> <li>22:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2152.codfw.wmnet with OS bookworm</li> <li>22:39 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2151.codfw.wmnet with OS bookworm</li> <li>22:38 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp1004.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:38 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp1006.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2155']</li> <li>22:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2154']</li> <li>22:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2153']</li> <li>22:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2152']</li> <li>22:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2151']</li> <li>22:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2151']</li> <li>22:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2152']</li> <li>22:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2153']</li> <li>22:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2154']</li> <li>22:37 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2155']</li> <li>22:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2153.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2155.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2152.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2151.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:35 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2154.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2155.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2153.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host wikikube-worker2155.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host wikikube-worker2153.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2155.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2154.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:24 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2153.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:23 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2152.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:23 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2151.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:22 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>22:22 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2151-55 to codfw - jhancock@cumin2002"</li> <li>22:22 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2151-55 to codfw - jhancock@cumin2002"</li> <li>22:18 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>22:16 jclark@cumin1002: START - Cookbook sre.hosts.provision for host mc-gp1005.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:16 jclark@cumin1002: START - Cookbook sre.hosts.provision for host mc-gp1004.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:16 jclark@cumin1002: START - Cookbook sre.hosts.provision for host mc-gp1006.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:14 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>22:14 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for mc-gp1004 - jclark@cumin1002"</li> <li>22:14 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for mc-gp1004 - jclark@cumin1002"</li> <li>22:10 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>21:43 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2150.codfw.wmnet with OS bookworm</li> <li>21:42 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:35 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:31 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2148.codfw.wmnet with OS bookworm</li> <li>21:31 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:31 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:27 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2147.codfw.wmnet with OS bookworm</li> <li>21:27 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:27 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:26 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2146.codfw.wmnet with OS bookworm</li> <li>21:26 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:26 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2149.codfw.wmnet with OS bookworm</li> <li>21:26 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:25 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:20 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:20 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:18 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>21:16 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2150.codfw.wmnet with reason: host reimage</li> <li>21:12 sukhe@puppetserver1001: conftool action : set/pooled=yes; selector: name=cp2031.codfw.wmnet [reason: PSU replaced]</li> <li>21:12 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2148.codfw.wmnet with reason: host reimage</li> <li>21:08 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2147.codfw.wmnet with reason: host reimage</li> <li>21:05 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2146.codfw.wmnet with reason: host reimage</li> <li>21:01 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2149.codfw.wmnet with reason: host reimage</li> <li>20:59 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2150.codfw.wmnet with reason: host reimage</li> <li>20:59 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2148.codfw.wmnet with reason: host reimage</li> <li>20:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2147.codfw.wmnet with reason: host reimage</li> <li>20:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2146.codfw.wmnet with reason: host reimage</li> <li>20:58 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2149.codfw.wmnet with reason: host reimage</li> <li>20:41 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2148.codfw.wmnet with OS bookworm</li> <li>20:41 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2150.codfw.wmnet with OS bookworm</li> <li>20:40 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2149.codfw.wmnet with OS bookworm</li> <li>20:40 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2147.codfw.wmnet with OS bookworm</li> <li>20:40 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2146.codfw.wmnet with OS bookworm</li> <li>20:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2150']</li> <li>20:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2149']</li> <li>20:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2148']</li> <li>20:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2147']</li> <li>20:39 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2146']</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2150']</li> <li>20:39 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2149']</li> <li>20:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2148']</li> <li>20:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2147']</li> <li>20:38 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2146']</li> <li>20:37 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2149.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:37 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2146.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2150.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2148.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:36 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2147.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:27 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2149.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:26 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host wikikube-worker2149.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:26 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2150.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:26 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2149.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:26 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2148.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2147.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:25 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2146.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>20:25 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:25 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2146-50 to codfw - jhancock@cumin2002"</li> <li>20:24 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2146-50 to codfw - jhancock@cumin2002"</li> <li>20:19 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>19:55 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc-gp2006.codfw.wmnet with OS bookworm</li> <li>19:55 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:41 brett: Remove RSA cert support from P:idp clients (icinga, karma, klaxon, librenms, orchestrator) (<a href="https://phabricator.wikimedia.org/T375569" class="extiw" title="phab:T375569">T375569</a>)</li> <li>18:10 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>18:10 elukey@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>18:06 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>18:03 sukhe: dummy authdns-update to test CR <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="10857508"><a class="external text" href="https://gerrit.wikimedia.org/r/q/10857508">10857508</a></span></li> <li>17:48 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc-gp2006.codfw.wmnet with reason: host reimage</li> <li>17:45 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on mc-gp2006.codfw.wmnet with reason: host reimage</li> <li>17:35 elukey@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - elukey@cumin1002"</li> <li>17:27 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host mc-gp2006.codfw.wmnet with OS bookworm</li> <li>17:17 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp2006.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:17 hnowlan: importing debs for mercurius-1.0.1</li> <li>17:15 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host mc-gp2006.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>17:14 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be2083.codfw.wmnet with reason: host reimage</li> <li>17:11 elukey@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2083.codfw.wmnet with reason: host reimage</li> <li>17:11 vriley@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:11 vriley@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt fransw1001 - vriley@cumin1002"</li> <li>17:11 vriley@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt fransw1001 - vriley@cumin1002"</li> <li>17:05 vriley@cumin1002: START - Cookbook sre.dns.netbox</li> <li>16:58 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>16:37 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:36 vriley@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:35 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:32 moritzm: remove ganeti1014 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>16:31 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>16:26 jclark@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:26 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:25 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>16:24 jclark@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:23 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:21 jclark@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:21 jclark@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for fransc1001 - jclark@cumin1002"</li> <li>16:20 jclark@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: added mgmt for fransc1001 - jclark@cumin1002"</li> <li>16:17 jclark@cumin1002: START - Cookbook sre.dns.netbox</li> <li>16:10 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db2136 gradually with 4 steps - cloned on db2236</li> <li>16:10 jclark@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:08 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:08 jclark@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>16:01 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs4010.ulsfo.wmnet</li> <li>15:59 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:58 vriley@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:57 mfossati@deploy2002: Finished deploy [airflow-dags/platform_eng@294093b]: remove section alignment image suggestions, now in section topics v1.0.0 (duration: 01m 23s)</li> <li>15:57 vriley@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>15:57 vriley@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt fransc1001 - vriley@cumin1002"</li> <li>15:57 vriley@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt fransc1001 - vriley@cumin1002"</li> <li>15:57 mfossati@deploy2002: Started deploy [airflow-dags/platform_eng@294093b]: remove section alignment image suggestions, now in section topics v1.0.0</li> <li>15:55 topranks: rebooting lvs4010 to verify new IPv6 sysctl's for RA processing work <a href="https://phabricator.wikimedia.org/T358260" class="extiw" title="phab:T358260">T358260</a></li> <li>15:55 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:25:00 on cr[3-4]-ulsfo with reason: prevent bgp alerts firing while lvs4010 is rebooted</li> <li>15:55 cmooney@cumin1002: START - Cookbook sre.hosts.downtime for 0:25:00 on cr[3-4]-ulsfo with reason: prevent bgp alerts firing while lvs4010 is rebooted</li> <li>15:55 cmooney@cumin1002: START - Cookbook sre.hosts.reboot-single for host lvs4010.ulsfo.wmnet</li> <li>15:53 vriley@cumin1002: START - Cookbook sre.dns.netbox</li> <li>15:51 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:50 vriley@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:48 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:48 vriley@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:43 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:42 vriley@cumin1002: START - Cookbook sre.hosts.provision for host fransc1001.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>15:31 moritzm: installing Linux 5.10.226 on bullseye hosts</li> <li>15:24 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db2136 gradually with 4 steps - cloned on db2236</li> <li>15:18 mutante: gitlab1004 - systemctl start wmf_auto_restart_ssh-gitlab (because it had failed with "Service ssh-gitlab not present or not running") but now it's just fine and exits with "No restart necessary" <a href="https://phabricator.wikimedia.org/T379166" class="extiw" title="phab:T379166">T379166</a></li> <li>15:13 elukey@cumin1002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>15:12 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087877" class="extiw" title="gerrit:1087877">Document available wbformatvalue options (T323778)</a> (duration: 38m 45s)</li> <li>15:07 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.clone (exit_code=0) of db2136.codfw.wmnet onto db2236.codfw.wmnet</li> <li>15:00 lucaswerkmeister-wmde@deploy2002: lucaswerkmeister-wmde: Continuing with sync</li> <li>14:59 lucaswerkmeister-wmde@deploy2002: lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1087877" class="extiw" title="gerrit:1087877">Document available wbformatvalue options (T323778)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:51 moritzm: installing php7.4 security updates</li> <li>14:50 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1046.eqiad.wmnet</li> <li>14:48 moritzm: installing usb.ids updates from Bookworm point release</li> <li>14:43 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1046.eqiad.wmnet</li> <li>14:42 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1046</li> <li>14:36 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1046</li> <li>14:33 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087877" class="extiw" title="gerrit:1087877">Document available wbformatvalue options (T323778)</a></li> <li>14:31 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085572" class="extiw" title="gerrit:1085572">Cleanup for logo related file</a> (duration: 15m 01s)</li> <li>14:31 vgutierrez@cumin1002: END (PASS) - Cookbook sre.dns.admin (exit_code=0) DNS admin: pool site eqiad for service: ncredir-addrs [reason: no reason specified, <a href="https://phabricator.wikimedia.org/T378453" class="extiw" title="phab:T378453">T378453</a>]</li> <li>14:31 vgutierrez@cumin1002: START - Cookbook sre.dns.admin DNS admin: pool site eqiad for service: ncredir-addrs [reason: no reason specified, <a href="https://phabricator.wikimedia.org/T378453" class="extiw" title="phab:T378453">T378453</a>]</li> <li>14:27 lucaswerkmeister-wmde@deploy2002: hamishz, lucaswerkmeister-wmde: Continuing with sync</li> <li>14:26 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1045.eqiad.wmnet</li> <li>14:20 sukhe@puppetserver1001: conftool action : set/pooled=no; selector: name=cp2031.codfw.wmnet</li> <li>14:19 sukhe: depool cp2031</li> <li>14:19 lucaswerkmeister-wmde@deploy2002: hamishz, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1085572" class="extiw" title="gerrit:1085572">Cleanup for logo related file</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:19 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1045.eqiad.wmnet</li> <li>14:16 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085572" class="extiw" title="gerrit:1085572">Cleanup for logo related file</a></li> <li>14:16 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1045</li> <li>14:14 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1045</li> <li>14:02 vgutierrez@cumin1002: END (PASS) - Cookbook sre.dns.admin (exit_code=0) DNS admin: depool site eqiad for service: ncredir-addrs [reason: no reason specified, <a href="https://phabricator.wikimedia.org/T378453" class="extiw" title="phab:T378453">T378453</a>]</li> <li>14:02 vgutierrez@cumin1002: START - Cookbook sre.dns.admin DNS admin: depool site eqiad for service: ncredir-addrs [reason: no reason specified, <a href="https://phabricator.wikimedia.org/T378453" class="extiw" title="phab:T378453">T378453</a>]</li> <li>13:52 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>13:52 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1044.eqiad.wmnet to cluster eqiad and group B</li> <li>13:47 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1044.eqiad.wmnet to cluster eqiad and group B</li> <li>13:44 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of dse-k8s-etcd1002.eqiad.wmnet to plain</li> <li>13:43 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:42 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:41 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of dse-k8s-etcd1002.eqiad.wmnet to plain</li> <li>13:28 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>13:27 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>13:27 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.drain-node (exit_code=99) for draining ganeti node ganeti1041.eqiad.wmnet</li> <li>13:27 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1041.eqiad.wmnet</li> <li>13:08 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of dse-k8s-etcd1002.eqiad.wmnet to drbd</li> <li>13:02 arnaudb@cumin1002: START - Cookbook sre.mysql.clone of db2136.codfw.wmnet onto db2236.codfw.wmnet</li> <li>12:58 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of dse-k8s-etcd1002.eqiad.wmnet to drbd</li> <li>12:56 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ml-etcd1001.eqiad.wmnet to plain</li> <li>12:56 arnaudb@cumin1002: dbctl commit (dc=all): 'Cloning db2136 in db2236 for <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a>', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70964">https://phabricator.wikimedia.org/P70964</a> and previous config saved to /var/cache/conftool/dbconfig/20241106-125648-arnaudb.json</li> <li>12:55 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ml-etcd1001.eqiad.wmnet to plain</li> <li>12:55 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.depool (exit_code=0) db2136 - depooling db2136 to clone on db2236</li> <li>12:55 arnaudb@cumin1002: START - Cookbook sre.mysql.depool db2136 - depooling db2136 to clone on db2236</li> <li>12:55 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2236.codfw.wmnet with reason: provisionning db2236.codfw.wmnet - <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>12:54 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2236.codfw.wmnet with reason: provisionning db2236.codfw.wmnet - <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>12:54 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2136.codfw.wmnet with reason: provisionning db2236.codfw.wmnet - <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>12:54 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2136.codfw.wmnet with reason: provisionning db2236.codfw.wmnet - <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>12:52 slyngs: IDP/CAS-SSO Enable Redis TGT backend</li> <li>12:52 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>12:52 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>12:50 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ml-etcd1001.eqiad.wmnet to drbd</li> <li>12:41 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ml-etcd1001.eqiad.wmnet to drbd</li> <li>12:40 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db1206 quickly with 2 steps - test <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1087895"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1087895/">1087895</a></span></li> <li>12:25 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db1206 quickly with 2 steps - test <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1087895"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1087895/">1087895</a></span></li> <li>12:23 arnaudb@cumin1002: dbctl commit (dc=all): 'db1206 depool to test cookbook hotfix on CR 1087895', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70960">https://phabricator.wikimedia.org/P70960</a> and previous config saved to /var/cache/conftool/dbconfig/20241106-122348-arnaudb.json</li> <li>12:23 marostegui: Migrate db1125 to MariaDB 10.6.20 <a href="https://phabricator.wikimedia.org/T378940" class="extiw" title="phab:T378940">T378940</a></li> <li>12:23 arnaudb@cumin1002: dbctl commit (dc=all): '"db1206 pending"', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70959">https://phabricator.wikimedia.org/P70959</a> and previous config saved to /var/cache/conftool/dbconfig/20241106-122318-arnaudb.json</li> <li>12:21 marostegui@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 6:00:00 on db2230.codfw.wmnet with reason: testing</li> <li>12:21 marostegui@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 6:00:00 on db2230.codfw.wmnet with reason: testing</li> <li>12:21 marostegui@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 6:00:00 on db1125.eqiad.wmnet with reason: testing</li> <li>12:21 marostegui@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 6:00:00 on db1125.eqiad.wmnet with reason: testing</li> <li>12:09 arnaudb@cumin1002: END (FAIL) - Cookbook sre.mysql.pool (exit_code=99) db1206 quickly with 2 steps - repool</li> <li>12:09 arnaudb@cumin1002: START - Cookbook sre.mysql.pool db1206 quickly with 2 steps - repool</li> <li>12:06 mvolz@deploy2002: helmfile [eqiad] DONE helmfile.d/services/citoid: apply</li> <li>12:06 mvolz@deploy2002: helmfile [eqiad] START helmfile.d/services/citoid: apply</li> <li>12:05 arnaudb@cumin1002: dbctl commit (dc=all): 'Depool db1206', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70957">https://phabricator.wikimedia.org/P70957</a> and previous config saved to /var/cache/conftool/dbconfig/20241106-120536-arnaudb.json</li> <li>12:03 mvolz@deploy2002: helmfile [codfw] DONE helmfile.d/services/citoid: apply</li> <li>12:03 mvolz@deploy2002: helmfile [codfw] START helmfile.d/services/citoid: apply</li> <li>12:02 mvolz@deploy2002: helmfile [staging] DONE helmfile.d/services/citoid: apply</li> <li>12:02 mvolz@deploy2002: helmfile [staging] START helmfile.d/services/citoid: apply</li> <li>11:37 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:37 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:32 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:31 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:30 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:30 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:30 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1041.eqiad.wmnet</li> <li>11:08 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1041.eqiad.wmnet</li> <li>10:50 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>10:43 fabfur: rolling out haproxykafka on all ULSFO cp hosts (<a class="external free" href="https://gerrit.wikimedia.org/r/c/operations/puppet/+/1087862">https://gerrit.wikimedia.org/r/c/operations/puppet/+/1087862</a>) (<a href="https://phabricator.wikimedia.org/T378578" class="extiw" title="phab:T378578">T378578</a>)</li> <li>10:43 elukey: depool maps1005 to test an nginx config - <a href="https://phabricator.wikimedia.org/T378944" class="extiw" title="phab:T378944">T378944</a></li> <li>10:41 jnuche@deploy2002: rebuilt and synchronized wikiversions files: group1 to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>10:32 XioNoX: push new pfw policies - <a href="https://phabricator.wikimedia.org/T379127" class="extiw" title="phab:T379127">T379127</a></li> <li>10:28 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ml-etcd1001.eqiad.wmnet to plain</li> <li>10:27 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ml-etcd1001.eqiad.wmnet to plain</li> <li>10:16 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>10:15 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>10:15 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>10:12 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>10:12 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.changedisk (exit_code=0) for changing disk type of ml-etcd1001.eqiad.wmnet to drbd</li> <li>09:59 jmm@cumin2002: START - Cookbook sre.ganeti.changedisk for changing disk type of ml-etcd1001.eqiad.wmnet to drbd</li> <li>09:59 jnuche@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087863" class="extiw" title="gerrit:1087863">Fix automatic category creations by FuzzyBot (T285463)</a> (duration: 08m 03s)</li> <li>09:55 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1044.eqiad.wmnet to cluster eqiad and group B</li> <li>09:54 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1044.eqiad.wmnet to cluster eqiad and group B</li> <li>09:54 jnuche@deploy2002: jnuche: Continuing with sync</li> <li>09:54 jnuche@deploy2002: jnuche: Backport for <a href="https://gerrit.wikimedia.org/r/1087863" class="extiw" title="gerrit:1087863">Fix automatic category creations by FuzzyBot (T285463)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>09:53 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1043.eqiad.wmnet to cluster eqiad and group B</li> <li>09:52 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1043.eqiad.wmnet to cluster eqiad and group B</li> <li>09:51 jnuche@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087863" class="extiw" title="gerrit:1087863">Fix automatic category creations by FuzzyBot (T285463)</a></li> <li>09:49 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1044.eqiad.wmnet</li> <li>09:41 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1044.eqiad.wmnet</li> <li>09:38 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>09:38 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1043.eqiad.wmnet</li> <li>09:31 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1043.eqiad.wmnet</li> <li>09:29 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1044</li> <li>09:28 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1044</li> <li>09:27 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1043</li> <li>09:25 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1043</li> <li>09:20 elukey@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>09:10 elukey@cumin2002: START - Cookbook sre.hosts.reimage for host ms-be2083.codfw.wmnet with OS bullseye</li> <li>08:56 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2083.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:46 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ms-be2083.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART</li> <li>08:12 volans: manually cleared /root/.ssh/known_hosts on the cumin hosts - <a href="https://phabricator.wikimedia.org/T336485" class="extiw" title="phab:T336485">T336485</a></li> <li>05:52 kart_: Updated cxserver to 2024-10-25-044319-production (<a href="https://phabricator.wikimedia.org/T377160" class="extiw" title="phab:T377160">T377160</a>, <a href="https://phabricator.wikimedia.org/T375102" class="extiw" title="phab:T375102">T375102</a>, <a href="https://phabricator.wikimedia.org/T371420" class="extiw" title="phab:T371420">T371420</a>)</li> <li>05:38 kartik@deploy2002: helmfile [eqiad] DONE helmfile.d/services/cxserver: apply</li> <li>05:38 kartik@deploy2002: helmfile [eqiad] START helmfile.d/services/cxserver: apply</li> <li>05:37 kartik@deploy2002: helmfile [codfw] DONE helmfile.d/services/cxserver: apply</li> <li>05:36 kartik@deploy2002: helmfile [codfw] START helmfile.d/services/cxserver: apply</li> <li>05:34 kartik@deploy2002: helmfile [staging] DONE helmfile.d/services/cxserver: apply</li> <li>05:33 kartik@deploy2002: helmfile [staging] START helmfile.d/services/cxserver: apply</li> <li>01:30 zabe@deploy2002: Finished scap sync-world: <a href="https://phabricator.wikimedia.org/T378260" class="extiw" title="phab:T378260">T378260</a> (duration: 07m 34s)</li> <li>01:23 zabe@deploy2002: Started scap sync-world: <a href="https://phabricator.wikimedia.org/T378260" class="extiw" title="phab:T378260">T378260</a></li> <li>00:44 ladsgroup@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) es1021 gradually with 4 steps - Maint over</li> <li>00:21 ryankemper: <a href="https://phabricator.wikimedia.org/T377594" class="extiw" title="phab:T377594">T377594</a> Merged <a class="external free" href="https://gerrit.wikimedia.org/r/c/operations/puppet/+/1087598">https://gerrit.wikimedia.org/r/c/operations/puppet/+/1087598</a>; ran puppet on `snapshot101[0-7]*`. These dumps should be re-enabled now</li> <li>00:02 ebernhardson@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087592" class="extiw" title="gerrit:1087592">TextPassDumper: refresh content address on failure (T377594)</a>, <a href="https://gerrit.wikimedia.org/r/1087593" class="extiw" title="gerrit:1087593">TextPassDumper: refresh content address on failure (T377594)</a> (duration: 08m 48s)</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-05" data-mw-thread-id="h-2024-11-05"><span data-mw-comment-start="" id="h-2024-11-05"></span>2024-11-05<span data-mw-comment-end="h-2024-11-05"></span></h2></div> <ul><li>23:59 ladsgroup@cumin1002: START - Cookbook sre.mysql.pool es1021 gradually with 4 steps - Maint over</li> <li>23:58 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2134.codfw.wmnet with OS bookworm</li> <li>23:58 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:57 ebernhardson@deploy2002: ebernhardson: Continuing with sync</li> <li>23:57 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:57 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2135.codfw.wmnet with OS bookworm</li> <li>23:57 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:57 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:56 ebernhardson@deploy2002: ebernhardson: Backport for <a href="https://gerrit.wikimedia.org/r/1087592" class="extiw" title="gerrit:1087592">TextPassDumper: refresh content address on failure (T377594)</a>, <a href="https://gerrit.wikimedia.org/r/1087593" class="extiw" title="gerrit:1087593">TextPassDumper: refresh content address on failure (T377594)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>23:56 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2132.codfw.wmnet with OS bookworm</li> <li>23:56 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:55 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2130.codfw.wmnet with OS bookworm</li> <li>23:54 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2133.codfw.wmnet with OS bookworm</li> <li>23:54 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host wikikube-worker2131.codfw.wmnet with OS bookworm</li> <li>23:54 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:53 ebernhardson@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087592" class="extiw" title="gerrit:1087592">TextPassDumper: refresh content address on failure (T377594)</a>, <a href="https://gerrit.wikimedia.org/r/1087593" class="extiw" title="gerrit:1087593">TextPassDumper: refresh content address on failure (T377594)</a></li> <li>23:50 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:44 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:39 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2134.codfw.wmnet with reason: host reimage</li> <li>23:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2132.codfw.wmnet with reason: host reimage</li> <li>23:30 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2131.codfw.wmnet with reason: host reimage</li> <li>23:26 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2135.codfw.wmnet with reason: host reimage</li> <li>23:23 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2130.codfw.wmnet with reason: host reimage</li> <li>23:19 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on wikikube-worker2133.codfw.wmnet with reason: host reimage</li> <li>23:18 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2135.codfw.wmnet with reason: host reimage</li> <li>23:18 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2134.codfw.wmnet with reason: host reimage</li> <li>23:17 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2132.codfw.wmnet with reason: host reimage</li> <li>23:16 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2131.codfw.wmnet with reason: host reimage</li> <li>23:16 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2130.codfw.wmnet with reason: host reimage</li> <li>23:16 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on wikikube-worker2133.codfw.wmnet with reason: host reimage</li> <li>23:00 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2135.codfw.wmnet with OS bookworm</li> <li>23:00 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2134.codfw.wmnet with OS bookworm</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2133.codfw.wmnet with OS bookworm</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2132.codfw.wmnet with OS bookworm</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2131.codfw.wmnet with OS bookworm</li> <li>22:58 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host wikikube-worker2130.codfw.wmnet with OS bookworm</li> <li>22:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2135']</li> <li>22:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2134']</li> <li>22:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2133']</li> <li>22:54 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2132']</li> <li>22:53 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2131']</li> <li>22:52 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['wikikube-worker2130']</li> <li>22:52 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2135']</li> <li>22:52 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2134']</li> <li>22:52 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2133']</li> <li>22:52 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2132']</li> <li>22:52 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2131']</li> <li>22:52 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['wikikube-worker2130']</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2135.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2134.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2132.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2130.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2133.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:42 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2131.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2135.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2134.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2133.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2132.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2131.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:31 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host wikikube-worker2130.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host wikikube-worker2134</li> <li>22:30 jhancock@cumin2002: END (FAIL) - Cookbook sre.network.configure-switch-interfaces (exit_code=99) for host wikikube-worker2135</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host wikikube-worker2133</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host wikikube-worker2132</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host wikikube-worker2131</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host wikikube-worker2130</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2135</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2134</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2133</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2132</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2131</li> <li>22:30 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2130</li> <li>22:29 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>22:29 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2130 to codfw - jhancock@cumin2002"</li> <li>22:29 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding wikikube-worker2130 to codfw - jhancock@cumin2002"</li> <li>22:29 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host wikikube-worker2132</li> <li>22:26 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>21:47 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087560" class="extiw" title="gerrit:1087560">AbstractProvider: Normalize top level config correctly (T379094)</a>, <a href="https://gerrit.wikimedia.org/r/1087561" class="extiw" title="gerrit:1087561">AbstractProvider: Normalize top level config correctly (T379094)</a> (duration: 12m 39s)</li> <li>21:34 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087560" class="extiw" title="gerrit:1087560">AbstractProvider: Normalize top level config correctly (T379094)</a>, <a href="https://gerrit.wikimedia.org/r/1087561" class="extiw" title="gerrit:1087561">AbstractProvider: Normalize top level config correctly (T379094)</a></li> <li>21:33 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087540" class="extiw" title="gerrit:1087540">cswiki: adding throttle rule for Editathon Czechoslovakia (T379060)</a> (duration: 31m 18s)</li> <li>21:11 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:06 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:02 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087540" class="extiw" title="gerrit:1087540">cswiki: adding throttle rule for Editathon Czechoslovakia (T379060)</a></li> <li>21:01 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>21:00 cmooney@cumin1002: END (PASS) - Cookbook sre.network.provision (exit_code=0) for device fasw2-c1b-eqiad.mgmt.eqiad.wmnet</li> <li>20:56 cmooney@cumin1002: END (PASS) - Cookbook sre.network.provision (exit_code=0) for device fasw2-c1a-eqiad.mgmt.eqiad.wmnet</li> <li>20:56 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>20:14 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:14 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add management record for fasw2-c1b-eqiad - cmooney@cumin1002"</li> <li>20:14 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add management record for fasw2-c1b-eqiad - cmooney@cumin1002"</li> <li>20:07 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>20:07 cmooney@cumin1002: START - Cookbook sre.network.provision for device fasw2-c1b-eqiad.mgmt.eqiad.wmnet</li> <li>20:02 cmooney@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:02 cmooney@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add management record for fasw2-c1a-eqiad - cmooney@cumin1002"</li> <li>20:02 cmooney@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add management record for fasw2-c1a-eqiad - cmooney@cumin1002"</li> <li>19:57 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>19:57 cmooney@cumin1002: START - Cookbook sre.network.provision for device fasw2-c1a-eqiad.mgmt.eqiad.wmnet</li> <li>19:56 cmooney@cumin1002: END (FAIL) - Cookbook sre.network.provision (exit_code=99) for device fasw2-c1a-eqiad.mgmt.eqiad.wmnet</li> <li>19:56 cmooney@cumin1002: START - Cookbook sre.network.provision for device fasw2-c1a-eqiad.mgmt.eqiad.wmnet</li> <li>19:52 cmooney@cumin1002: END (FAIL) - Cookbook sre.network.provision (exit_code=99) for device fasw2-c1a-eqiad.mgmt.eqiad.wmnet</li> <li>19:52 cmooney@cumin1002: START - Cookbook sre.network.provision for device fasw2-c1a-eqiad.mgmt.eqiad.wmnet</li> <li>19:20 eileen: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="26d8013c"><a class="external text" href="https://gerrit.wikimedia.org/r/q/26d8013c">26d8013c</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="65a8de90"><a class="external text" href="https://gerrit.wikimedia.org/r/q/65a8de90">65a8de90</a></span></li> <li>18:45 cmooney@cumin1002: START - Cookbook sre.dns.netbox</li> <li>18:10 Amir1: gradual delete of thumbs in fawiki local images in both dcs</li> <li>18:00 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1021 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70948">https://phabricator.wikimedia.org/P70948</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-180013-ladsgroup.json</li> <li>18:00 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1021.eqiad.wmnet with reason: Maintenance</li> <li>17:59 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1021.eqiad.wmnet with reason: Maintenance</li> <li>17:58 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1028 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70947">https://phabricator.wikimedia.org/P70947</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-175851-ladsgroup.json</li> <li>17:55 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>17:55 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>17:43 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1028', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70946">https://phabricator.wikimedia.org/P70946</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-174344-ladsgroup.json</li> <li>17:42 cdanis@deploy2002: helmfile [codfw] DONE helmfile.d/services/chart-renderer: apply</li> <li>17:41 cdanis@deploy2002: helmfile [codfw] START helmfile.d/services/chart-renderer: apply</li> <li>17:41 cdanis@deploy2002: helmfile [eqiad] DONE helmfile.d/services/chart-renderer: apply</li> <li>17:41 cdanis@deploy2002: helmfile [eqiad] START helmfile.d/services/chart-renderer: apply</li> <li>17:39 cdanis@deploy2002: helmfile [staging] DONE helmfile.d/services/chart-renderer: apply</li> <li>17:39 cdanis@deploy2002: helmfile [staging] START helmfile.d/services/chart-renderer: apply</li> <li>17:36 akosiaris@deploy2002: helmfile [codfw] DONE helmfile.d/services/rest-gateway: apply</li> <li>17:36 akosiaris@deploy2002: helmfile [codfw] START helmfile.d/services/rest-gateway: apply</li> <li>17:34 akosiaris@deploy2002: helmfile [eqiad] DONE helmfile.d/services/rest-gateway: apply</li> <li>17:34 akosiaris@deploy2002: helmfile [eqiad] START helmfile.d/services/rest-gateway: apply</li> <li>17:33 akosiaris@deploy2002: helmfile [staging] DONE helmfile.d/services/rest-gateway: apply</li> <li>17:33 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/rest-gateway: apply</li> <li>17:32 cdanis@deploy2002: helmfile [staging] DONE helmfile.d/services/chart-renderer: apply</li> <li>17:32 cdanis@deploy2002: helmfile [staging] START helmfile.d/services/chart-renderer: apply</li> <li>17:28 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1028', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70945">https://phabricator.wikimedia.org/P70945</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-172837-ladsgroup.json</li> <li>17:13 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1028 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70943">https://phabricator.wikimedia.org/P70943</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-171330-ladsgroup.json</li> <li>17:06 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1028 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70942">https://phabricator.wikimedia.org/P70942</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-170636-ladsgroup.json</li> <li>17:06 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1028.eqiad.wmnet with reason: Maintenance</li> <li>17:06 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1028.eqiad.wmnet with reason: Maintenance</li> <li>17:06 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1031 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70941">https://phabricator.wikimedia.org/P70941</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-170609-ladsgroup.json</li> <li>16:51 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1031', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70940">https://phabricator.wikimedia.org/P70940</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-165103-ladsgroup.json</li> <li>16:37 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087507" class="extiw" title="gerrit:1087507">Fixup paths to moved resources (T379080)</a> (duration: 08m 02s)</li> <li>16:35 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1031', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70939">https://phabricator.wikimedia.org/P70939</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-163556-ladsgroup.json</li> <li>16:34 cdanis@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:32 lucaswerkmeister-wmde@deploy2002: lucaswerkmeister-wmde: Continuing with sync</li> <li>16:32 lucaswerkmeister-wmde@deploy2002: lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1087507" class="extiw" title="gerrit:1087507">Fixup paths to moved resources (T379080)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>16:32 cdanis@cumin1002: START - Cookbook sre.dns.netbox</li> <li>16:29 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087507" class="extiw" title="gerrit:1087507">Fixup paths to moved resources (T379080)</a></li> <li>16:20 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1031 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70938">https://phabricator.wikimedia.org/P70938</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-162048-ladsgroup.json</li> <li>16:14 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1031 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70937">https://phabricator.wikimedia.org/P70937</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-161455-ladsgroup.json</li> <li>16:14 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1031.eqiad.wmnet with reason: Maintenance</li> <li>16:14 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1031.eqiad.wmnet with reason: Maintenance</li> <li>16:13 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1033 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70936">https://phabricator.wikimedia.org/P70936</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-161340-ladsgroup.json</li> <li>16:01 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host pc1017.eqiad.wmnet with OS bookworm</li> <li>16:00 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>15:58 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1033', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70935">https://phabricator.wikimedia.org/P70935</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-155833-ladsgroup.json</li> <li>15:54 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>15:54 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.drain-node (exit_code=99) for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>15:54 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1014.eqiad.wmnet</li> <li>15:53 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1042.eqiad.wmnet to cluster eqiad and group B</li> <li>15:51 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1042.eqiad.wmnet to cluster eqiad and group B</li> <li>15:51 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1041.eqiad.wmnet to cluster eqiad and group B</li> <li>15:50 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1041.eqiad.wmnet to cluster eqiad and group B</li> <li>15:48 moritzm: remove ganeti1013 from active ganeti nodes <a href="https://phabricator.wikimedia.org/T378921" class="extiw" title="phab:T378921">T378921</a></li> <li>15:47 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1013.eqiad.wmnet</li> <li>15:43 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1033', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70934">https://phabricator.wikimedia.org/P70934</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-154326-ladsgroup.json</li> <li>15:40 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on pc1017.eqiad.wmnet with reason: host reimage</li> <li>15:37 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on pc1017.eqiad.wmnet with reason: host reimage</li> <li>15:32 hashar: Switched PCC workers to Java 17 via <a class="external free" href="https://horizon.wikimedia.org/project/prefixpuppet/?tab=prefix_puppet__puppet-pcc-worker">https://horizon.wikimedia.org/project/prefixpuppet/?tab=prefix_puppet__puppet-pcc-worker</a> # <a href="https://phabricator.wikimedia.org/T359795" class="extiw" title="phab:T359795">T359795</a></li> <li>15:28 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1033 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70933">https://phabricator.wikimedia.org/P70933</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-152819-ladsgroup.json</li> <li>15:27 hashar: Switched deployment-deploy04.deployment-prep.eqiad1.wikimedia.cloud to Java 17 # <a href="https://phabricator.wikimedia.org/T359795" class="extiw" title="phab:T359795">T359795</a></li> <li>15:21 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1033 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70932">https://phabricator.wikimedia.org/P70932</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-152139-ladsgroup.json</li> <li>15:21 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1033.eqiad.wmnet with reason: Maintenance</li> <li>15:21 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1033.eqiad.wmnet with reason: Maintenance</li> <li>15:21 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1026 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70931">https://phabricator.wikimedia.org/P70931</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-152114-ladsgroup.json</li> <li>15:20 arnaudb@cumin1002: START - Cookbook sre.hosts.reimage for host pc1017.eqiad.wmnet with OS bookworm</li> <li>15:18 hashar: Switched WMCS integration instances from Java 11 to Java 17 via Horizon project wide config. That was forgotten in <a href="https://phabricator.wikimedia.org/T359795" class="extiw" title="phab:T359795">T359795</a> and blocks today Jenkins upgrade ( <a href="https://phabricator.wikimedia.org/T379059" class="extiw" title="phab:T379059">T379059</a> )</li> <li>15:15 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host pc1017.eqiad.wmnet with OS bookworm</li> <li>15:06 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1026', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70929">https://phabricator.wikimedia.org/P70929</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-150607-ladsgroup.json</li> <li>15:02 cdanis@deploy2002: helmfile [eqiad] DONE helmfile.d/services/chart-renderer: apply</li> <li>15:02 cdanis@deploy2002: helmfile [eqiad] START helmfile.d/services/chart-renderer: apply</li> <li>15:02 cdanis@deploy2002: helmfile [codfw] DONE helmfile.d/services/chart-renderer: apply</li> <li>15:01 cdanis@deploy2002: helmfile [codfw] START helmfile.d/services/chart-renderer: apply</li> <li>15:01 hashar: Upgrading CI Jenkins | <a href="https://phabricator.wikimedia.org/T379059" class="extiw" title="phab:T379059">T379059</a></li> <li>14:53 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on pc1017.eqiad.wmnet with reason: host reimage</li> <li>14:51 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1026', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70928">https://phabricator.wikimedia.org/P70928</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-145059-ladsgroup.json</li> <li>14:50 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on pc1017.eqiad.wmnet with reason: host reimage</li> <li>14:48 jnuche@deploy2002: rebuilt and synchronized wikiversions files: group0 to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>14:44 cdanis@deploy2002: helmfile [staging] DONE helmfile.d/services/chart-renderer: apply</li> <li>14:44 cdanis@deploy2002: helmfile [staging] START helmfile.d/services/chart-renderer: apply</li> <li>14:35 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1026 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70927">https://phabricator.wikimedia.org/P70927</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-143552-ladsgroup.json</li> <li>14:34 arnaudb@cumin1002: START - Cookbook sre.hosts.reimage for host pc1017.eqiad.wmnet with OS bookworm</li> <li>14:33 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host pc1017.eqiad.wmnet with OS bookworm</li> <li>away: UTC afternoon deploys done</li> <li>14:30 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1026 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70926">https://phabricator.wikimedia.org/P70926</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-142959-ladsgroup.json</li> <li>14:29 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1026.eqiad.wmnet with reason: Maintenance</li> <li>14:29 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1026.eqiad.wmnet with reason: Maintenance</li> <li>14:29 vgutierrez: upload liberica 0.3 to apt.wm.o (bookworm-wikimedia)</li> <li>14:28 tgr@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087455" class="extiw" title="gerrit:1087455">JsonConfig: Disable TrackGlobalJsonLinks to avoid missing table errors (T379067)</a> (duration: 17m 24s)</li> <li>14:24 tgr@deploy2002: tgr: Continuing with sync</li> <li>14:16 tgr@deploy2002: tgr: Backport for <a href="https://gerrit.wikimedia.org/r/1087455" class="extiw" title="gerrit:1087455">JsonConfig: Disable TrackGlobalJsonLinks to avoid missing table errors (T379067)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:12 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on pc1017.eqiad.wmnet with reason: host reimage</li> <li>14:11 tgr@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087455" class="extiw" title="gerrit:1087455">JsonConfig: Disable TrackGlobalJsonLinks to avoid missing table errors (T379067)</a></li> <li>14:10 akosiaris@deploy2002: helmfile [eqiad] DONE helmfile.d/services/rest-gateway: apply</li> <li>14:10 akosiaris@deploy2002: helmfile [eqiad] START helmfile.d/services/rest-gateway: apply</li> <li>14:09 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on pc1017.eqiad.wmnet with reason: host reimage</li> <li>14:08 moritzm: installing PHP 7.4 security updates on bullseye (as packaged in Debian)</li> <li>14:08 akosiaris@deploy2002: helmfile [codfw] DONE helmfile.d/services/rest-gateway: apply</li> <li>14:07 akosiaris@deploy2002: helmfile [codfw] START helmfile.d/services/rest-gateway: apply</li> <li>14:07 akosiaris@deploy2002: helmfile [staging] DONE helmfile.d/services/rest-gateway: apply</li> <li>14:07 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/rest-gateway: apply</li> <li>13:57 moritzm: installed libapache2-mod-auth-openidc bugfix updates from Bookworm point release</li> <li>13:54 arnaudb: reimage pc1017 <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a></li> <li>13:53 arnaudb@cumin1002: START - Cookbook sre.hosts.reimage for host pc1017.eqiad.wmnet with OS bookworm</li> <li>13:52 akosiaris@deploy2002: helmfile [staging] DONE helmfile.d/services/rest-gateway: apply</li> <li>13:52 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/rest-gateway: apply</li> <li>13:44 akosiaris@deploy2002: helmfile [staging] DONE helmfile.d/services/rest-gateway: apply</li> <li>13:44 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/rest-gateway: apply</li> <li>13:42 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:42 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>13:41 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/rest-gateway: apply</li> <li>13:39 akosiaris@deploy2002: helmfile [staging] DONE helmfile.d/services/rest-gateway: apply</li> <li>13:34 moritzm: imported jenkins 2.479.1 to thirdparty/ci for bullseye-wikimedia <a href="https://phabricator.wikimedia.org/T379059" class="extiw" title="phab:T379059">T379059</a></li> <li>13:29 akosiaris@deploy2002: helmfile [staging] START helmfile.d/services/rest-gateway: apply</li> <li>13:16 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>13:16 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on pc1017.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T378068" class="extiw" title="phab:T378068">T378068</a>, host is not pooled</li> <li>13:10 cmooney@cumin1002: END (PASS) - Cookbook sre.netbox.update-extras (exit_code=0) rolling restart_daemons on A:netbox</li> <li>13:10 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1042.eqiad.wmnet</li> <li>13:10 cmooney@cumin1002: START - Cookbook sre.netbox.update-extras rolling restart_daemons on A:netbox</li> <li>13:09 cmooney@cumin1002: END (PASS) - Cookbook sre.netbox.update-extras (exit_code=0) rolling restart_daemons on A:netbox-canary</li> <li>13:09 cmooney@cumin1002: START - Cookbook sre.netbox.update-extras rolling restart_daemons on A:netbox-canary</li> <li>13:08 moritzm: installing php7.4 security updates on remaining non-wikikube servers <a href="https://phabricator.wikimedia.org/T378173" class="extiw" title="phab:T378173">T378173</a></li> <li>13:03 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1042.eqiad.wmnet</li> <li>12:56 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1041.eqiad.wmnet</li> <li>12:50 kharlan@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087424" class="extiw" title="gerrit:1087424">Revert^2 "temp accounts: Enable temp account creation on second-round pilots" (T378336)</a> (duration: 11m 46s)</li> <li>12:49 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1041.eqiad.wmnet</li> <li>12:46 kharlan@deploy2002: kharlan: Continuing with sync</li> <li>12:42 kharlan@deploy2002: kharlan: Backport for <a href="https://gerrit.wikimedia.org/r/1087424" class="extiw" title="gerrit:1087424">Revert^2 "temp accounts: Enable temp account creation on second-round pilots" (T378336)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>12:40 fnegri@cumin1002: END (PASS) - Cookbook sre.wikireplicas.update-views (exit_code=0)</li> <li>12:39 kharlan@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087424" class="extiw" title="gerrit:1087424">Revert^2 "temp accounts: Enable temp account creation on second-round pilots" (T378336)</a></li> <li>12:35 fnegri@cumin1002: START - Cookbook sre.wikireplicas.update-views</li> <li>12:35 fnegri@cumin1002: END (FAIL) - Cookbook sre.wikireplicas.update-views (exit_code=93)</li> <li>12:35 fnegri@cumin1002: START - Cookbook sre.wikireplicas.update-views</li> <li>12:34 fnegri@cumin1002: END (FAIL) - Cookbook sre.wikireplicas.update-views (exit_code=93)</li> <li>12:34 fnegri@cumin1002: START - Cookbook sre.wikireplicas.update-views</li> <li>12:33 urbanecm: eswiki,x1: `delete from growthexperiments_link_recommendations where gelr_page=10598298;` (to verify updates are flowing in; <a href="https://phabricator.wikimedia.org/T378983" class="extiw" title="phab:T378983">T378983</a>)</li> <li>12:33 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1013.eqiad.wmnet</li> <li>12:33 urbanecm: mwmaint2002: kill all instances of refreshLinkRecommendation (<a href="https://phabricator.wikimedia.org/T378983" class="extiw" title="phab:T378983">T378983</a>)</li> <li>12:32 jmm@cumin2002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1013.eqiad.wmnet</li> <li>12:28 jmm@cumin2002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1013.eqiad.wmnet</li> <li>12:23 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087407" class="extiw" title="gerrit:1087407">CirrusSearch: Disable updating weighted tags via EventBus (T378983 T377150)</a> (duration: 07m 39s)</li> <li>12:18 marostegui@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 6:00:00 on db1125.eqiad.wmnet with reason: testing</li> <li>12:18 marostegui@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 6:00:00 on db1125.eqiad.wmnet with reason: testing</li> <li>12:18 marostegui@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 6:00:00 on db2230.codfw.wmnet with reason: testing</li> <li>12:17 marostegui@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 6:00:00 on db2230.codfw.wmnet with reason: testing</li> <li>12:16 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087407" class="extiw" title="gerrit:1087407">CirrusSearch: Disable updating weighted tags via EventBus (T378983 T377150)</a></li> <li>12:10 jnuche@deploy2002: Finished scap sync-world: testwikis to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a> (duration: 07m 43s)</li> <li>12:04 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1040.eqiad.wmnet to cluster eqiad and group B</li> <li>12:02 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1040.eqiad.wmnet to cluster eqiad and group B</li> <li>12:02 jnuche@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>12:01 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1040.eqiad.wmnet</li> <li>11:57 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1040.eqiad.wmnet</li> <li>11:53 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1042</li> <li>11:53 jnuche@deploy2002: rebuilt and synchronized wikiversions files: group0 to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>11:53 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1029 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70922">https://phabricator.wikimedia.org/P70922</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-115301-ladsgroup.json</li> <li>11:52 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1042</li> <li>11:49 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1041</li> <li>11:47 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1041</li> <li>11:47 jmm@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host ganeti1040</li> <li>11:46 jmm@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host ganeti1040</li> <li>11:39 jnuche@deploy2002: Finished scap sync-world: testwikis to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a> (duration: 36m 28s)</li> <li>11:37 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1029', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70921">https://phabricator.wikimedia.org/P70921</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-113754-ladsgroup.json</li> <li>11:22 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1029', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70920">https://phabricator.wikimedia.org/P70920</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-112246-ladsgroup.json</li> <li>11:07 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1029 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70919">https://phabricator.wikimedia.org/P70919</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-110739-ladsgroup.json</li> <li>11:02 jnuche@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>11:01 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1029 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70918">https://phabricator.wikimedia.org/P70918</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-110139-ladsgroup.json</li> <li>11:01 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1029.eqiad.wmnet with reason: Maintenance</li> <li>11:01 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1029.eqiad.wmnet with reason: Maintenance</li> <li>11:01 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1032 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70917">https://phabricator.wikimedia.org/P70917</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-110115-ladsgroup.json</li> <li>10:46 jnuche@deploy2002: Installing scap version "4.121.0" for 209 hosts</li> <li>10:46 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1032', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70916">https://phabricator.wikimedia.org/P70916</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-104608-ladsgroup.json</li> <li>10:44 jnuche@deploy2002: install-world aborted: (no justification provided) (duration: 03m 09s)</li> <li>10:41 jnuche@deploy2002: Installing scap version "4.121.0" for 209 hosts</li> <li>10:41 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>10:40 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>10:31 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1032', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70915">https://phabricator.wikimedia.org/P70915</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-103101-ladsgroup.json</li> <li>10:15 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance es1032 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70914">https://phabricator.wikimedia.org/P70914</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-101553-ladsgroup.json</li> <li>10:11 elukey: set proxy timeouts of docker registry's nginx instances from 300s to 180s - <a href="https://phabricator.wikimedia.org/T378618" class="extiw" title="phab:T378618">T378618</a></li> <li>10:09 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling es1032 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70913">https://phabricator.wikimedia.org/P70913</a> and previous config saved to /var/cache/conftool/dbconfig/20241105-100953-ladsgroup.json</li> <li>10:09 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es1032.eqiad.wmnet with reason: Maintenance</li> <li>10:09 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es1032.eqiad.wmnet with reason: Maintenance</li> <li>10:07 vgutierrez@cumin1002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host lvs1013.eqiad.wmnet with OS bookworm</li> <li>10:00 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1171.eqiad.wmnet with reason: Maintenance</li> <li>10:00 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1171.eqiad.wmnet with reason: Maintenance</li> <li>09:49 vgutierrez@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on lvs1013.eqiad.wmnet with reason: host reimage</li> <li>09:45 vgutierrez@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on lvs1013.eqiad.wmnet with reason: host reimage</li> <li>09:33 vgutierrez@cumin1002: START - Cookbook sre.hosts.reimage for host lvs1013.eqiad.wmnet with OS bookworm</li> <li>09:31 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10 days, 0:00:00 on pc1013.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T373037" class="extiw" title="phab:T373037">T373037</a>, host is not pooled</li> <li>09:31 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 10 days, 0:00:00 on pc1013.eqiad.wmnet with reason: <a href="https://phabricator.wikimedia.org/T373037" class="extiw" title="phab:T373037">T373037</a>, host is not pooled</li> <li>09:22 jnuche@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>09:21 _joe_: restarted rsyslog on deploy2002 <a href="https://phabricator.wikimedia.org/T379044" class="extiw" title="phab:T379044">T379044</a></li> <li>08:57 tchanders@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087373" class="extiw" title="gerrit:1087373">Revert "temp accounts: Enable temp account creation on second-round pilots"</a></li> <li>08:24 vgutierrez: uploaded ipip-multiqueue-optimizer 0.3+deb12u1 to apt.wm.o (bookworm)</li> <li>08:10 tchanders@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087195" class="extiw" title="gerrit:1087195">temp accounts: Enable temp account creation on second-round pilots (T378336)</a></li> <li>08:06 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 2828</li> <li>08:03 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 2828</li> <li>08:03 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 14593</li> <li>07:55 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'configure' for AS: 14593</li> <li>07:39 ayounsi@cumin1002: END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 11414</li> <li>07:39 ayounsi@cumin1002: START - Cookbook sre.network.peering with action 'email' for AS: 11414</li> <li>05:10 mwpresync@deploy2002: Pruned MediaWiki: 1.43.0-wmf.27 (duration: 10m 37s)</li> <li>04:03 mwpresync@deploy2002: Started scap sync-world: testwikis to 1.44.0-wmf.2 refs <a href="https://phabricator.wikimedia.org/T375661" class="extiw" title="phab:T375661">T375661</a></li> <li>00:10 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp2006.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>00:10 rzl@deploy2002: Finished scap sync-world: <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1085506"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1085506/">1085506</a></span> (duration: 02m 50s)</li> <li>00:08 rzl@deploy2002: Started scap sync-world: <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1085506"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1085506/">1085506</a></span></li> <li>00:04 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host mc-gp2006.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-04" data-mw-thread-id="h-2024-11-04"><span data-mw-comment-start="" id="h-2024-11-04"></span>2024-11-04<span data-mw-comment-end="h-2024-11-04"></span></h2></div> <ul><li>23:56 jhancock@cumin2002: END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host mc-gp2006</li> <li>23:56 jhancock@cumin2002: START - Cookbook sre.network.configure-switch-interfaces for host mc-gp2006</li> <li>23:56 jhancock@cumin2002: END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host mc-gp2006.codfw.wmnet with OS bookworm</li> <li>23:18 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc-gp2005.codfw.wmnet with OS bookworm</li> <li>23:18 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:18 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:17 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc-gp2004.codfw.wmnet with OS bookworm</li> <li>23:17 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>23:15 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>22:59 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc-gp2005.codfw.wmnet with reason: host reimage</li> <li>22:56 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc-gp2004.codfw.wmnet with reason: host reimage</li> <li>22:53 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on mc-gp2005.codfw.wmnet with reason: host reimage</li> <li>22:53 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on mc-gp2004.codfw.wmnet with reason: host reimage</li> <li>22:35 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host mc-gp2006.codfw.wmnet with OS bookworm</li> <li>22:35 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host mc-gp2005.codfw.wmnet with OS bookworm</li> <li>22:35 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host mc-gp2004.codfw.wmnet with OS bookworm</li> <li>22:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['mc-gp2006']</li> <li>22:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['mc-gp2005']</li> <li>22:33 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['mc-gp2004']</li> <li>22:33 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['mc-gp2006']</li> <li>22:32 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['mc-gp2005']</li> <li>22:32 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['mc-gp2004']</li> <li>22:30 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp2006.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:29 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:29 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host mc-gp2004.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:22 damilare: civicrm upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="31f5cbdb"><a class="external text" href="https://gerrit.wikimedia.org/r/q/31f5cbdb">31f5cbdb</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="26d8013c"><a class="external text" href="https://gerrit.wikimedia.org/r/q/26d8013c">26d8013c</a></span></li> <li>22:22 damilare: SmashPig upgraded from <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="be47dddd"><a class="external text" href="https://gerrit.wikimedia.org/r/q/be47dddd">be47dddd</a></span> to <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="601405dc"><a class="external text" href="https://gerrit.wikimedia.org/r/q/601405dc">601405dc</a></span></li> <li>22:17 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host mc-gp2006.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:17 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host mc-gp2005.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:17 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host mc-gp2004.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>22:16 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>22:16 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding mc-gp2004 to codfw - jhancock@cumin2002"</li> <li>22:16 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding mc-gp2004 to codfw - jhancock@cumin2002"</li> <li>22:12 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>22:01 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kubestage2003.codfw.wmnet with OS bookworm</li> <li>22:00 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>22:00 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1226 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70912">https://phabricator.wikimedia.org/P70912</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-220026-ladsgroup.json</li> <li>22:00 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:58 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kubestage2004.codfw.wmnet with OS bookworm</li> <li>21:58 jhancock@cumin2002: END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:57 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002"</li> <li>21:45 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1226', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70911">https://phabricator.wikimedia.org/P70911</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-214519-ladsgroup.json</li> <li>away: UTC late deploys done</li> <li>21:41 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubestage2003.codfw.wmnet with reason: host reimage</li> <li>21:41 tgr@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087207" class="extiw" title="gerrit:1087207">Set Flow to read-only on remaining phase 0 wikis (T377990)</a> (duration: 08m 40s)</li> <li>21:38 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubestage2004.codfw.wmnet with reason: host reimage</li> <li>21:36 tgr@deploy2002: tgr, kemayo: Continuing with sync</li> <li>21:35 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on kubestage2003.codfw.wmnet with reason: host reimage</li> <li>21:35 jhancock@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on kubestage2004.codfw.wmnet with reason: host reimage</li> <li>21:35 tgr@deploy2002: tgr, kemayo: Backport for <a href="https://gerrit.wikimedia.org/r/1087207" class="extiw" title="gerrit:1087207">Set Flow to read-only on remaining phase 0 wikis (T377990)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>21:32 tgr@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087207" class="extiw" title="gerrit:1087207">Set Flow to read-only on remaining phase 0 wikis (T377990)</a></li> <li>21:31 eevans@cumin1002: END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) for nodes matching sessionstore2*: Apply openjdk upgrade (11.0.25+9-1~deb11u1) - eevans@cumin1002</li> <li>21:30 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1226', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70910">https://phabricator.wikimedia.org/P70910</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-213012-ladsgroup.json</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host kubestage2004.codfw.wmnet with OS bookworm</li> <li>21:17 jhancock@cumin2002: START - Cookbook sre.hosts.reimage for host kubestage2003.codfw.wmnet with OS bookworm</li> <li>21:15 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['kubestage2004']</li> <li>21:15 jhancock@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['kubestage2003']</li> <li>21:15 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['kubestage2004']</li> <li>21:15 jhancock@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['kubestage2003']</li> <li>21:15 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1226 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70909">https://phabricator.wikimedia.org/P70909</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-211505-ladsgroup.json</li> <li>21:14 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host kubestage2004.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:14 jhancock@cumin2002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host kubestage2003.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:14 eevans@cumin1002: START - Cookbook sre.cassandra.roll-restart for nodes matching sessionstore2*: Apply openjdk upgrade (11.0.25+9-1~deb11u1) - eevans@cumin1002</li> <li>21:08 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1226 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70908">https://phabricator.wikimedia.org/P70908</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-210800-ladsgroup.json</li> <li>21:07 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1226.eqiad.wmnet with reason: Maintenance</li> <li>21:07 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1226.eqiad.wmnet with reason: Maintenance</li> <li>21:05 eevans@cumin1002: END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) for nodes matching sessionstore1*: Apply openjdk upgrade (11.0.25+9-1~deb11u1) - eevans@cumin1002</li> <li>21:03 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host kubestage2004.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:03 jhancock@cumin2002: START - Cookbook sre.hosts.provision for host kubestage2003.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED</li> <li>21:02 jhancock@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>21:02 jhancock@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding kubestage2003 to codfw - jhancock@cumin2002"</li> <li>21:02 jhancock@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: adding kubestage2003 to codfw - jhancock@cumin2002"</li> <li>21:02 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1216.eqiad.wmnet with reason: Maintenance</li> <li>21:02 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1216.eqiad.wmnet with reason: Maintenance</li> <li>21:02 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1214 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70907">https://phabricator.wikimedia.org/P70907</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-210224-ladsgroup.json</li> <li>20:59 jhancock@cumin2002: START - Cookbook sre.dns.netbox</li> <li>20:47 eevans@cumin1002: START - Cookbook sre.cassandra.roll-restart for nodes matching sessionstore1*: Apply openjdk upgrade (11.0.25+9-1~deb11u1) - eevans@cumin1002</li> <li>20:47 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1214', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70906">https://phabricator.wikimedia.org/P70906</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-204717-ladsgroup.json</li> <li>20:35 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts aqs1013.eqiad.wmnet</li> <li>20:35 eevans@cumin1002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>20:35 eevans@cumin1002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: aqs1013.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - eevans@cumin1002"</li> <li>20:32 eevans@cumin1002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: aqs1013.eqiad.wmnet decommissioned, removing all IPs except the asset tag one - eevans@cumin1002"</li> <li>20:32 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1214', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70905">https://phabricator.wikimedia.org/P70905</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-203210-ladsgroup.json</li> <li>20:27 eevans@cumin1002: START - Cookbook sre.dns.netbox</li> <li>20:26 swfrench-wmf: zero-replica "migration" releases created for all shellbox instances - <a href="https://phabricator.wikimedia.org/T375243" class="extiw" title="phab:T375243">T375243</a></li> <li>20:23 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/shellbox-video: apply</li> <li>20:23 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/shellbox-video: apply</li> <li>20:22 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/shellbox-timeline: apply</li> <li>20:22 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/shellbox-timeline: apply</li> <li>20:22 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/shellbox-media: apply</li> <li>20:21 eevans@cumin1002: START - Cookbook sre.hosts.decommission for hosts aqs1013.eqiad.wmnet</li> <li>20:21 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/shellbox-media: apply</li> <li>20:21 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/shellbox-constraints: apply</li> <li>20:20 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/shellbox-constraints: apply</li> <li>20:20 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/shellbox: apply</li> <li>20:19 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/shellbox: apply</li> <li>20:17 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1214 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70904">https://phabricator.wikimedia.org/P70904</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-201703-ladsgroup.json</li> <li>20:09 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1214 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70903">https://phabricator.wikimedia.org/P70903</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-200905-ladsgroup.json</li> <li>20:08 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1214.eqiad.wmnet with reason: Maintenance</li> <li>20:08 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1214.eqiad.wmnet with reason: Maintenance</li> <li>20:08 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1211 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70902">https://phabricator.wikimedia.org/P70902</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-200840-ladsgroup.json</li> <li>20:00 urbanecm@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087231" class="extiw" title="gerrit:1087231">Message: Downgrade exception on bool/null param to warning (T378876)</a> (duration: 09m 12s)</li> <li>19:55 urbanecm@deploy2002: urbanecm: Continuing with sync</li> <li>19:54 urbanecm@deploy2002: urbanecm: Backport for <a href="https://gerrit.wikimedia.org/r/1087231" class="extiw" title="gerrit:1087231">Message: Downgrade exception on bool/null param to warning (T378876)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>19:53 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1211', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70901">https://phabricator.wikimedia.org/P70901</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-195333-ladsgroup.json</li> <li>19:51 urbanecm@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1087231" class="extiw" title="gerrit:1087231">Message: Downgrade exception on bool/null param to warning (T378876)</a></li> <li>19:38 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1211', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70900">https://phabricator.wikimedia.org/P70900</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-193826-ladsgroup.json</li> <li>19:23 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1211 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70899">https://phabricator.wikimedia.org/P70899</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-192319-ladsgroup.json</li> <li>19:23 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/shellbox-video: apply</li> <li>19:22 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/shellbox-video: apply</li> <li>19:22 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/shellbox-timeline: apply</li> <li>19:21 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/shellbox-timeline: apply</li> <li>19:21 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/shellbox-media: apply</li> <li>19:20 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/shellbox-media: apply</li> <li>19:19 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/shellbox-constraints: apply</li> <li>19:18 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/shellbox-constraints: apply</li> <li>19:18 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/shellbox: apply</li> <li>19:17 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/shellbox: apply</li> <li>19:15 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1211 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70898">https://phabricator.wikimedia.org/P70898</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-191519-ladsgroup.json</li> <li>19:15 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1211.eqiad.wmnet with reason: Maintenance</li> <li>19:14 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1211.eqiad.wmnet with reason: Maintenance</li> <li>19:14 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1209 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70897">https://phabricator.wikimedia.org/P70897</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-191454-ladsgroup.json</li> <li>19:09 swfrench@deploy2002: helmfile [codfw] DONE helmfile.d/services/shellbox-syntaxhighlight: apply</li> <li>19:09 swfrench@deploy2002: helmfile [codfw] START helmfile.d/services/shellbox-syntaxhighlight: apply</li> <li>19:04 swfrench@deploy2002: helmfile [eqiad] DONE helmfile.d/services/shellbox-syntaxhighlight: apply</li> <li>19:03 swfrench@deploy2002: helmfile [eqiad] START helmfile.d/services/shellbox-syntaxhighlight: apply</li> <li>18:59 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1209', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70896">https://phabricator.wikimedia.org/P70896</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-185947-ladsgroup.json</li> <li>18:58 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/shellbox-video: apply</li> <li>18:57 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/shellbox-video: apply</li> <li>18:57 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/shellbox-timeline: apply</li> <li>18:56 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/shellbox-timeline: apply</li> <li>18:56 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/shellbox-syntaxhighlight: apply</li> <li>18:56 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/shellbox-syntaxhighlight: apply</li> <li>18:56 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/shellbox-media: apply</li> <li>18:55 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/shellbox-media: apply</li> <li>18:55 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/shellbox-constraints: apply</li> <li>18:54 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/shellbox-constraints: apply</li> <li>18:54 swfrench@deploy2002: helmfile [staging] DONE helmfile.d/services/shellbox: apply</li> <li>18:53 swfrench@deploy2002: helmfile [staging] START helmfile.d/services/shellbox: apply</li> <li>18:47 vgutierrez@cumin1002: END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 1 day, 0:00:00 on lvs1013.eqiad.wmnet with reason: known issues with liberica-hcforwarder and ipip-multiqueue-optimizer</li> <li>18:47 vgutierrez@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on lvs1013.eqiad.wmnet with reason: known issues with liberica-hcforwarder and ipip-multiqueue-optimizer</li> <li>18:44 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1209', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70895">https://phabricator.wikimedia.org/P70895</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-184440-ladsgroup.json</li> <li>18:41 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for lvs2013.codfw.wmnet</li> <li>18:41 sukhe@cumin1002: START - Cookbook sre.hosts.remove-downtime for lvs2013.codfw.wmnet</li> <li>18:41 sukhe@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on lvs2013.codfw.wmnet with reason: vgutierrez</li> <li>18:41 sukhe@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on lvs2013.codfw.wmnet with reason: vgutierrez</li> <li>18:29 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1209 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70894">https://phabricator.wikimedia.org/P70894</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-182933-ladsgroup.json</li> <li>18:25 vgutierrez@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host lvs1013.eqiad.wmnet with OS bookworm</li> <li>18:21 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1209 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70893">https://phabricator.wikimedia.org/P70893</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-182140-ladsgroup.json</li> <li>18:21 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1209.eqiad.wmnet with reason: Maintenance</li> <li>18:21 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1209.eqiad.wmnet with reason: Maintenance</li> <li>18:21 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1203 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70892">https://phabricator.wikimedia.org/P70892</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-182125-ladsgroup.json</li> <li>18:06 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1203', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70891">https://phabricator.wikimedia.org/P70891</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-180618-ladsgroup.json</li> <li>18:01 vgutierrez@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on lvs1013.eqiad.wmnet with reason: host reimage</li> <li>17:56 vgutierrez@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on lvs1013.eqiad.wmnet with reason: host reimage</li> <li>17:51 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1203', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70890">https://phabricator.wikimedia.org/P70890</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-175111-ladsgroup.json</li> <li>17:43 vgutierrez@cumin1002: START - Cookbook sre.hosts.reimage for host lvs1013.eqiad.wmnet with OS bookworm</li> <li>17:43 vgutierrez: upload liberica 0.2 to apt.wm.o (bookworm) - <a href="https://phabricator.wikimedia.org/T377127" class="extiw" title="phab:T377127">T377127</a></li> <li>17:37 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host sretest2001.codfw.wmnet with OS bookworm</li> <li>17:36 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1203 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70889">https://phabricator.wikimedia.org/P70889</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-173604-ladsgroup.json</li> <li>17:35 vgutierrez@cumin1002: END (FAIL) - Cookbook sre.puppet.migrate-host (exit_code=99) for host lvs1013.eqiad.wmnet</li> <li>17:35 vgutierrez@cumin1002: START - Cookbook sre.puppet.migrate-host for host lvs1013.eqiad.wmnet</li> <li>17:26 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1203 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70888">https://phabricator.wikimedia.org/P70888</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-172638-ladsgroup.json</li> <li>17:26 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1203.eqiad.wmnet with reason: Maintenance</li> <li>17:26 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1203.eqiad.wmnet with reason: Maintenance</li> <li>17:26 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1192 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70887">https://phabricator.wikimedia.org/P70887</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-172612-ladsgroup.json</li> <li>17:23 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on sretest2001.codfw.wmnet with reason: host reimage</li> <li>17:20 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on sretest2001.codfw.wmnet with reason: host reimage</li> <li>17:11 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1192', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70886">https://phabricator.wikimedia.org/P70886</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-171105-ladsgroup.json</li> <li>17:07 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host sretest2001.codfw.wmnet with OS bookworm</li> <li>17:06 pt1979@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>17:04 pt1979@cumin2002: START - Cookbook sre.dns.netbox</li> <li>16:59 vgutierrez@cumin1002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host lvs1013.eqiad.wmnet with OS bookworm</li> <li>16:55 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1192', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70885">https://phabricator.wikimedia.org/P70885</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-165558-ladsgroup.json</li> <li>16:40 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1192 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70883">https://phabricator.wikimedia.org/P70883</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-164051-ladsgroup.json</li> <li>16:37 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host sretest2001.codfw.wmnet with OS bookworm</li> <li>16:31 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1192 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70882">https://phabricator.wikimedia.org/P70882</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-163129-ladsgroup.json</li> <li>16:31 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1192.eqiad.wmnet with reason: Maintenance</li> <li>16:31 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1192.eqiad.wmnet with reason: Maintenance</li> <li>16:31 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1178 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70881">https://phabricator.wikimedia.org/P70881</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-163104-ladsgroup.json</li> <li>16:23 pt1979@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on sretest2001.codfw.wmnet with reason: host reimage</li> <li>16:21 pt1979@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on sretest2001.codfw.wmnet with reason: host reimage</li> <li>16:15 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1178', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70880">https://phabricator.wikimedia.org/P70880</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-161557-ladsgroup.json</li> <li>16:15 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>16:14 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>16:14 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>16:12 arnaudb@cumin1002: END (PASS) - Cookbook sre.mysql.clone (exit_code=0) of db2135.codfw.wmnet onto db2235.codfw.wmnet</li> <li>16:07 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply</li> <li>16:06 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>16:06 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on db2160.codfw.wmnet with reason: cloning db2135@db2235</li> <li>16:05 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 3:00:00 on db2160.codfw.wmnet with reason: cloning db2135@db2235</li> <li>16:05 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply</li> <li>16:05 pt1979@cumin2002: START - Cookbook sre.hosts.reimage for host sretest2001.codfw.wmnet with OS bookworm</li> <li>16:02 arnaudb@cumin1002: START - Cookbook sre.mysql.clone of db2135.codfw.wmnet onto db2235.codfw.wmnet</li> <li>16:01 pt1979@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>16:00 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1178', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70879">https://phabricator.wikimedia.org/P70879</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-160050-ladsgroup.json</li> <li>16:00 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on db[2135,2235].codfw.wmnet with reason: cloning db2135@db2235</li> <li>16:00 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 3:00:00 on db[2135,2235].codfw.wmnet with reason: cloning db2135@db2235</li> <li>15:58 pt1979@cumin2002: START - Cookbook sre.dns.netbox</li> <li>15:54 vgutierrez@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on lvs1013.eqiad.wmnet with reason: host reimage</li> <li>15:51 vgutierrez@cumin1002: START - Cookbook sre.hosts.downtime for 2:00:00 on lvs1013.eqiad.wmnet with reason: host reimage</li> <li>15:47 pt1979@cumin2002: END (ERROR) - Cookbook sre.dns.netbox (exit_code=97)</li> <li>15:46 pt1979@cumin2002: START - Cookbook sre.dns.netbox</li> <li>15:45 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1178 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70878">https://phabricator.wikimedia.org/P70878</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-154543-ladsgroup.json</li> <li>15:40 vgutierrez@cumin1002: START - Cookbook sre.hosts.reimage for host lvs1013.eqiad.wmnet with OS bookworm</li> <li>15:36 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1178 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70877">https://phabricator.wikimedia.org/P70877</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-153613-ladsgroup.json</li> <li>15:36 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1178.eqiad.wmnet with reason: Maintenance</li> <li>15:35 vgutierrez: upload liberica 0.1 to apt.wm.o (bookworm) - <a href="https://phabricator.wikimedia.org/T377127" class="extiw" title="phab:T377127">T377127</a></li> <li>15:35 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1178.eqiad.wmnet with reason: Maintenance</li> <li>15:35 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1177 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70876">https://phabricator.wikimedia.org/P70876</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-153548-ladsgroup.json</li> <li>15:29 sukhe: running authdns-update to move CN traffic to eqsin from ulsfo: <a href="https://phabricator.wikimedia.org/T378744" class="extiw" title="phab:T378744">T378744</a></li> <li>15:20 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1177', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70874">https://phabricator.wikimedia.org/P70874</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-152041-ladsgroup.json</li> <li>15:05 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1177', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70873">https://phabricator.wikimedia.org/P70873</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-150534-ladsgroup.json</li> <li>14:50 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1177 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70872">https://phabricator.wikimedia.org/P70872</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-145027-ladsgroup.json</li> <li>14:41 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1177 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70871">https://phabricator.wikimedia.org/P70871</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-144101-ladsgroup.json</li> <li>14:40 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1177.eqiad.wmnet with reason: Maintenance</li> <li>14:40 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1177.eqiad.wmnet with reason: Maintenance</li> <li>14:40 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1172 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70870">https://phabricator.wikimedia.org/P70870</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-144037-ladsgroup.json</li> <li>14:38 Lucas_WMDE: UTC afternoon backport+config window done</li> <li>14:36 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1084765" class="extiw" title="gerrit:1084765">Exclude affiliates from P&E dashboard integration for CampaignEvents Extension (T377252)</a> (duration: 23m 39s)</li> <li>14:28 lucaswerkmeister-wmde@deploy2002: mhorsey, lucaswerkmeister-wmde: Continuing with sync</li> <li>14:25 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70869">https://phabricator.wikimedia.org/P70869</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-142530-ladsgroup.json</li> <li>14:24 moritzm: uploaded php7.4 7.4.33-1+0~20221108.73+debian10~1.gbpa00350a+wmf10u2+icu67u3 to component/icu67 (backports of latest security fixes to our PHP 7.4 build)</li> <li>14:23 lucaswerkmeister-wmde@deploy2002: mhorsey, lucaswerkmeister-wmde: Backport for <a href="https://gerrit.wikimedia.org/r/1084765" class="extiw" title="gerrit:1084765">Exclude affiliates from P&E dashboard integration for CampaignEvents Extension (T377252)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>14:12 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1084765" class="extiw" title="gerrit:1084765">Exclude affiliates from P&E dashboard integration for CampaignEvents Extension (T377252)</a></li> <li>14:10 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70868">https://phabricator.wikimedia.org/P70868</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-141023-ladsgroup.json</li> <li>13:55 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1172 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70867">https://phabricator.wikimedia.org/P70867</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-135516-ladsgroup.json</li> <li>13:51 marostegui: Start schema change on redacteddb1001:s8 <a href="https://phabricator.wikimedia.org/T367856" class="extiw" title="phab:T367856">T367856</a> (this will make replication in s8 lag for around 2-3 days)</li> <li>13:50 marostegui@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet with reason: Schema change <a href="https://phabricator.wikimedia.org/T367856" class="extiw" title="phab:T367856">T367856</a></li> <li>13:50 marostegui@cumin1002: START - Cookbook sre.hosts.downtime for 3 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet with reason: Schema change <a href="https://phabricator.wikimedia.org/T367856" class="extiw" title="phab:T367856">T367856</a></li> <li>13:46 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1172 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70866">https://phabricator.wikimedia.org/P70866</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-134605-ladsgroup.json</li> <li>13:45 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1172.eqiad.wmnet with reason: Maintenance</li> <li>13:45 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1172.eqiad.wmnet with reason: Maintenance</li> <li>13:40 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1171.eqiad.wmnet with reason: Maintenance</li> <li>13:40 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1171.eqiad.wmnet with reason: Maintenance</li> <li>13:40 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1167 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70865">https://phabricator.wikimedia.org/P70865</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-134021-ladsgroup.json</li> <li>13:25 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1039.eqiad.wmnet to cluster eqiad and group B</li> <li>13:25 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1167', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70864">https://phabricator.wikimedia.org/P70864</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-132513-ladsgroup.json</li> <li>13:24 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1039.eqiad.wmnet to cluster eqiad and group B</li> <li>13:11 Dreamy_Jazz: Started slow MediaModeration scan for commonswiki to be scanning as close to upload as possible - <a class="external free" href="https://wikitech.wikimedia.org/wiki/MediaModeration">https://wikitech.wikimedia.org/wiki/MediaModeration</a></li> <li>13:10 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1167', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70862">https://phabricator.wikimedia.org/P70862</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-131006-ladsgroup.json</li> <li>13:06 Dreamy_Jazz: Started MediaModeration scan on all wikis other than s4 (commonswiki + testcommonswiki) - <a class="external free" href="https://wikitech.wikimedia.org/wiki/MediaModeration">https://wikitech.wikimedia.org/wiki/MediaModeration</a></li> <li>12:55 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db1167 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70861">https://phabricator.wikimedia.org/P70861</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-125459-ladsgroup.json</li> <li>12:49 XioNoX: deploy "Add temporary LVS community for liberica test" - <a href="https://phabricator.wikimedia.org/T378453" class="extiw" title="phab:T378453">T378453</a></li> <li>12:45 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db1167 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70860">https://phabricator.wikimedia.org/P70860</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-124533-ladsgroup.json</li> <li>12:45 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet,clouddb[1016,1020].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance</li> <li>12:45 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet,clouddb[1016,1020].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance</li> <li>12:45 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1167.eqiad.wmnet with reason: Maintenance</li> <li>12:44 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1167.eqiad.wmnet with reason: Maintenance</li> <li>12:35 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1052.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>12:34 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-articlequality' for release 'main' .</li> <li>12:24 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1052.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>12:22 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-draftquality' for release 'main' .</li> <li>12:22 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-drafttopic' for release 'main' .</li> <li>12:20 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-editquality-reverted' for release 'main' .</li> <li>12:19 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-editquality-goodfaith' for release 'main' .</li> <li>12:19 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-editquality-damaging' for release 'main' .</li> <li>12:11 jmm@cumin2002: END (FAIL) - Cookbook sre.ganeti.addnode (exit_code=99) for new host ganeti1039.eqiad.wmnet to cluster eqiad and group B</li> <li>12:11 jmm@cumin2002: START - Cookbook sre.ganeti.addnode for new host ganeti1039.eqiad.wmnet to cluster eqiad and group B</li> <li>12:10 isaranto@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'revscoring-articletopic' for release 'main' .</li> <li>12:08 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1039.eqiad.wmnet</li> <li>12:08 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1051.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>12:01 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1039.eqiad.wmnet</li> <li>11:58 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1051.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:56 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1050.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:55 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2227 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70859">https://phabricator.wikimedia.org/P70859</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-115514-ladsgroup.json</li> <li>11:45 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1050.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:44 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1049.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:40 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2227', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70858">https://phabricator.wikimedia.org/P70858</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-114008-ladsgroup.json</li> <li>11:34 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1049.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:25 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2227', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70857">https://phabricator.wikimedia.org/P70857</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-112501-ladsgroup.json</li> <li>11:22 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1048.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:12 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1048.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:09 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2227 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70856">https://phabricator.wikimedia.org/P70856</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-110953-ladsgroup.json</li> <li>11:05 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1047.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>11:01 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db2227 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70855">https://phabricator.wikimedia.org/P70855</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-110141-ladsgroup.json</li> <li>11:01 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2227.codfw.wmnet with reason: Maintenance</li> <li>11:01 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2227.codfw.wmnet with reason: Maintenance</li> <li>11:01 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2194 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70854">https://phabricator.wikimedia.org/P70854</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-110113-ladsgroup.json</li> <li>10:54 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1047.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:52 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1046.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:48 XioNoX: eqiad: Prefer Lumen to reach ATT - <a href="https://phabricator.wikimedia.org/T377844" class="extiw" title="phab:T377844">T377844</a></li> <li>10:46 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2194', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70853">https://phabricator.wikimedia.org/P70853</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-104606-ladsgroup.json</li> <li>10:42 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1046.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:41 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1045.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:41 moritzm: installing libtool updates from Bookworm point release</li> <li>10:31 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1045.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:31 moritzm: installing libseccomp updates from Bookworm point release</li> <li>10:31 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1043.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:30 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2194', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70852">https://phabricator.wikimedia.org/P70852</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-103059-ladsgroup.json</li> <li>10:20 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1043.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:17 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1042.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:15 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2194 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70851">https://phabricator.wikimedia.org/P70851</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-101552-ladsgroup.json</li> <li>10:08 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db2194 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70850">https://phabricator.wikimedia.org/P70850</a> and previous config saved to /var/cache/conftool/dbconfig/20241104-100813-ladsgroup.json</li> <li>10:08 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2194.codfw.wmnet with reason: Maintenance</li> <li>10:07 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2194.codfw.wmnet with reason: Maintenance</li> <li>10:06 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1042.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:02 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>10:01 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2139.codfw.wmnet with reason: Maintenance</li> <li>10:01 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2139.codfw.wmnet with reason: Maintenance</li> <li>09:57 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>09:56 volans: deploying spicerack v8.15.2 to cumin[12]002</li> <li>09:55 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1040.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>09:50 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1040.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>09:42 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1039.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>09:37 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1039.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>09:07 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 13 hosts with reason: reboots for nftables</li> <li>09:06 jmm@cumin2002: START - Cookbook sre.hosts.downtime for 1:00:00 on 13 hosts with reason: reboots for nftables</li> <li>09:06 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on ganeti1045.eqiad.wmnet with reason: reboots for nftables</li> <li>09:06 jmm@cumin2002: START - Cookbook sre.hosts.downtime for 1:00:00 on ganeti1045.eqiad.wmnet with reason: reboots for nftables</li> <li>09:04 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1039.eqiad.wmnet</li> <li>08:59 jmm@cumin2002: START - Cookbook sre.hosts.reboot-single for host ganeti1039.eqiad.wmnet</li> <li>08:57 elukey@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>08:57 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>08:51 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>08:50 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>08:23 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti2014.codfw.wmnet</li> <li>08:23 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>08:23 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2014.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>08:22 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2014.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>08:21 arnaudb@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on db2239.codfw.wmnet with reason: waiting for productionnization <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>08:21 arnaudb@cumin1002: START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on db2239.codfw.wmnet with reason: waiting for productionnization <a href="https://phabricator.wikimedia.org/T373579" class="extiw" title="phab:T373579">T373579</a></li> <li>08:16 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>08:15 XioNoX: push Drop labtestwikitech return traffic term to eqiad routers - CR1083589</li> <li>08:12 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti2014.codfw.wmnet</li> <li>08:11 jmm@cumin2002: END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts ganeti2013.codfw.wmnet</li> <li>08:11 jmm@cumin2002: END (PASS) - Cookbook sre.dns.netbox (exit_code=0)</li> <li>08:11 jmm@cumin2002: END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2013.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>08:09 jmm@cumin2002: START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: ganeti2013.codfw.wmnet decommissioned, removing all IPs except the asset tag one - jmm@cumin2002"</li> <li>08:06 brouberol@deploy2002: helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'.</li> <li>08:05 brouberol@deploy2002: helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'.</li> <li>08:03 jmm@cumin2002: START - Cookbook sre.dns.netbox</li> <li>07:59 jmm@cumin2002: START - Cookbook sre.hosts.decommission for hosts ganeti2013.codfw.wmnet</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-02" data-mw-thread-id="h-2024-11-02"><span data-mw-comment-start="" id="h-2024-11-02"></span>2024-11-02<span data-mw-comment-end="h-2024-11-02"></span></h2></div> <ul><li>15:48 lucaswerkmeister-wmde@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085922" class="extiw" title="gerrit:1085922">Remove 'mainpage' from $wgForceUIMsgAsContentMsg for Wikidata (T184386)</a> (duration: 12m 09s)</li> <li>15:44 lucaswerkmeister-wmde@deploy2002: lucaswerkmeister-wmde, ladsgroup: Continuing with sync</li> <li>15:38 lucaswerkmeister-wmde@deploy2002: lucaswerkmeister-wmde, ladsgroup: Backport for <a href="https://gerrit.wikimedia.org/r/1085922" class="extiw" title="gerrit:1085922">Remove 'mainpage' from $wgForceUIMsgAsContentMsg for Wikidata (T184386)</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:36 lucaswerkmeister-wmde@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085922" class="extiw" title="gerrit:1085922">Remove 'mainpage' from $wgForceUIMsgAsContentMsg for Wikidata (T184386)</a></li> <li>15:26 reedy@deploy2002: Finished scap sync-world: use statemnts (duration: 07m 13s)</li> <li>15:19 reedy@deploy2002: Started scap sync-world: use statemnts</li> <li>15:13 reedy@deploy2002: Synchronized wmf-config/: Comment updates (duration: 07m 31s)</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="2024-11-01" data-mw-thread-id="h-2024-11-01"><span data-mw-comment-start="" id="h-2024-11-01"></span>2024-11-01<span data-mw-comment-end="h-2024-11-01"></span></h2></div> <ul><li>20:27 bking@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-presto1016.eqiad.wmnet with OS bullseye</li> <li>19:47 inflatador: bking@an-presto[1016:1020].eqiad.wmnet temporarily install perccli to check disk status without requiring reboot <a href="https://phabricator.wikimedia.org/T374924" class="extiw" title="phab:T374924">T374924</a></li> <li>19:34 bking@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-presto1016.eqiad.wmnet with reason: host reimage</li> <li>19:31 bking@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on an-presto1016.eqiad.wmnet with reason: host reimage</li> <li>19:16 bking@cumin2002: START - Cookbook sre.hosts.reimage for host an-presto1016.eqiad.wmnet with OS bullseye</li> <li>19:12 bking@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['an-presto1017.eqiad.wmnet']</li> <li>19:07 bking@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['an-presto1016.eqiad.wmnet']</li> <li>19:02 bking@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-presto1017.eqiad.wmnet']</li> <li>18:56 bking@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-presto1016.eqiad.wmnet']</li> <li>18:56 bking@cumin2002: END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['an-presto1017.eqiad.wmnet']</li> <li>18:56 bking@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-presto1017.eqiad.wmnet']</li> <li>18:51 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:51 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:51 vriley@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1052.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:47 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1051.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:46 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1050.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:46 vriley@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1052.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:46 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:46 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:44 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1049.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:44 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:44 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:43 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1048.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:42 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:42 vriley@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:41 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1051.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:41 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1050.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1046.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:40 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1047.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:39 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1049.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:39 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:39 vriley@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:38 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1045.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:38 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1048.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:35 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:35 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1046.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:35 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1047.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:35 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:34 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1043.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:34 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1042.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:34 jclark@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:33 vriley@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:33 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1045.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:33 vriley@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>18:32 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1040.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:29 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1043.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:29 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1042.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:29 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1041.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:26 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1040.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:25 jclark@cumin1002: END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ganeti1039.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:19 jclark@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1039.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART</li> <li>18:11 bking@cumin2002: END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['an-presto1018.eqiad.wmnet']</li> <li>18:10 bking@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-presto1018.eqiad.wmnet']</li> <li>18:09 bking@cumin2002: END (PASS) - Cookbook sre.puppet.renew-cert (exit_code=0) for an-presto1020.eqiad.wmnet: Renew puppet certificate - bking@cumin2002</li> <li>18:07 dancy@deploy2002: Installation of scap version "4.120.0" completed for 1 hosts</li> <li>18:07 bking@cumin2002: START - Cookbook sre.puppet.renew-cert for an-presto1020.eqiad.wmnet: Renew puppet certificate - bking@cumin2002</li> <li>18:06 dancy@deploy2002: Installing scap version "4.120.0" for 1 hosts</li> <li>18:04 bking@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>17:00 Dreamy_Jazz: Ran `/usr/local/bin/foreachwikiindblist /srv/mediawiki/dblists/all.dblist extensions/WikimediaEvents/maintenance/UpdatePeriodicMetrics.php --verbose`</li> <li>16:36 bking@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-presto1020.eqiad.wmnet with reason: host reimage</li> <li>16:33 bking@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on an-presto1020.eqiad.wmnet with reason: host reimage</li> <li>16:18 bking@cumin2002: START - Cookbook sre.hosts.reimage for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>16:17 mvernon@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 16:00:00 on thanos-be2003.codfw.wmnet with reason: give it time for sde1 fs to backfill</li> <li>16:17 mvernon@cumin2002: START - Cookbook sre.hosts.downtime for 2 days, 16:00:00 on thanos-be2003.codfw.wmnet with reason: give it time for sde1 fs to backfill</li> <li>16:16 mvernon@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 16:00:00 on db2239.codfw.wmnet with reason: not yet in production</li> <li>16:16 mvernon@cumin2002: START - Cookbook sre.hosts.downtime for 2 days, 16:00:00 on db2239.codfw.wmnet with reason: not yet in production</li> <li>16:05 bking@cumin2002: END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['an-presto1020.eqiad.wmnet']</li> <li>16:05 thcipriani@deploy2002: Finished scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085597" class="extiw" title="gerrit:1085597">Revert "Dummy commit for testing"</a> (duration: 07m 46s)</li> <li>16:00 thcipriani@deploy2002: thcipriani: Continuing with sync</li> <li>16:00 thcipriani@deploy2002: thcipriani: Backport for <a href="https://gerrit.wikimedia.org/r/1085597" class="extiw" title="gerrit:1085597">Revert "Dummy commit for testing"</a> synced to the testservers (<a class="external free" href="https://wikitech.wikimedia.org/wiki/Mwdebug">https://wikitech.wikimedia.org/wiki/Mwdebug</a>)</li> <li>15:57 thcipriani@deploy2002: Started scap sync-world: Backport for <a href="https://gerrit.wikimedia.org/r/1085597" class="extiw" title="gerrit:1085597">Revert "Dummy commit for testing"</a></li> <li>15:55 bking@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-presto1020.eqiad.wmnet']</li> <li>15:55 bking@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>15:19 mvernon@cumin2002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-be2003.codfw.wmnet</li> <li>15:05 mvernon@cumin2002: START - Cookbook sre.hosts.reboot-single for host thanos-be2003.codfw.wmnet</li> <li>14:54 bking@cumin2002: START - Cookbook sre.hosts.reimage for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>14:40 bking@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>14:29 bking@cumin2002: START - Cookbook sre.hosts.reimage for host an-presto1020.eqiad.wmnet with OS bullseye</li> <li>14:27 bking@cumin2002: END (ERROR) - Cookbook sre.hosts.reimage (exit_code=93) for host an-presto1020.eqiad.wmnet with OS bookworm</li> <li>14:06 ladsgroup@cumin1002: END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db2190 gradually with 4 steps - Maint over</li> <li>13:55 bking@cumin2002: START - Cookbook sre.hosts.reimage for host an-presto1020.eqiad.wmnet with OS bookworm</li> <li>13:43 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>13:43 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>13:38 elukey@cumin1002: END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>13:33 elukey@cumin1002: START - Cookbook sre.hosts.provision for host ganeti1044.mgmt.eqiad.wmnet with chassis set policy FORCE_RESTART</li> <li>13:20 ladsgroup@cumin1002: START - Cookbook sre.mysql.pool db2190 gradually with 4 steps - Maint over</li> <li>12:43 cmooney@cumin1002: END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1025.eqiad.wmnet</li> <li>12:43 cmooney@cumin1002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1025.eqiad.wmnet</li> <li>12:43 cmooney@cumin1002: END (FAIL) - Cookbook sre.ganeti.drain-node (exit_code=99) for draining ganeti node ganeti1025.eqiad.wmnet</li> <li>12:43 cmooney@cumin1002: START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1025.eqiad.wmnet</li> <li>12:42 cmooney@cumin1002: END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1025.eqiad.wmnet</li> <li>12:28 cmooney@cumin1002: START - Cookbook sre.hosts.reboot-single for host ganeti1025.eqiad.wmnet</li> <li>12:28 topranks: rebooting ganeti1025 as VMs are unresponsive and will not shutdown or move</li> <li>10:38 kevinbazira@deploy2002: helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'experimental' for release 'main' .</li> <li>off: sudo cumin -b4 "A:cp and A:magru" "run-puppet-agent" to pick up CR <span class="plainlinks" style="font-family: Consolas, Liberation Mono, Courier, monospace; text-decoration: none;" data-gerrit-id="1085569"><a class="external text" href="https://gerrit.wikimedia.org/r/c/1085569/">1085569</a></span></li> <li>02:25 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2198.codfw.wmnet with reason: Maintenance</li> <li>02:24 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2198.codfw.wmnet with reason: Maintenance</li> <li>02:24 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2195 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70840">https://phabricator.wikimedia.org/P70840</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-022447-ladsgroup.json</li> <li>02:09 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2195', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70839">https://phabricator.wikimedia.org/P70839</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-020940-ladsgroup.json</li> <li>01:59 bking@cumin2002: END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-presto1019.eqiad.wmnet with OS bullseye</li> <li>01:54 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2195', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70838">https://phabricator.wikimedia.org/P70838</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-015433-ladsgroup.json</li> <li>01:42 urandom: Decommissioning Cassandra/aqs1013-{a,b} — <a href="https://phabricator.wikimedia.org/T378725" class="extiw" title="phab:T378725">T378725</a></li> <li>01:41 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on aqs1013.eqiad.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T378725" class="extiw" title="phab:T378725">T378725</a></li> <li>01:40 eevans@cumin1002: START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on aqs1013.eqiad.wmnet with reason: Decommissioning — <a href="https://phabricator.wikimedia.org/T378725" class="extiw" title="phab:T378725">T378725</a></li> <li>01:39 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2195 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70837">https://phabricator.wikimedia.org/P70837</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-013926-ladsgroup.json</li> <li>01:39 eevans@cumin1002: END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for aqs1022.eqiad.wmnet</li> <li>01:39 eevans@cumin1002: START - Cookbook sre.hosts.remove-downtime for aqs1022.eqiad.wmnet</li> <li>01:31 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db2195 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70836">https://phabricator.wikimedia.org/P70836</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-013102-ladsgroup.json</li> <li>01:30 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2195.codfw.wmnet with reason: Maintenance</li> <li>01:30 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2195.codfw.wmnet with reason: Maintenance</li> <li>01:30 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2181 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70835">https://phabricator.wikimedia.org/P70835</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-013035-ladsgroup.json</li> <li>01:25 bking@cumin2002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-presto1019.eqiad.wmnet with reason: host reimage</li> <li>01:22 bking@cumin2002: START - Cookbook sre.hosts.downtime for 2:00:00 on an-presto1019.eqiad.wmnet with reason: host reimage</li> <li>01:15 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2181', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70834">https://phabricator.wikimedia.org/P70834</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-011528-ladsgroup.json</li> <li>01:07 bking@cumin2002: START - Cookbook sre.hosts.reimage for host an-presto1019.eqiad.wmnet with OS bullseye</li> <li>01:00 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2181', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70833">https://phabricator.wikimedia.org/P70833</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-010021-ladsgroup.json</li> <li>00:54 bking@cumin2002: START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-presto1019.eqiad.wmnet']</li> <li>00:54 bking@cumin2002: END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['an-presto1019.eqiad.wmnet']</li> <li>00:45 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2181 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70832">https://phabricator.wikimedia.org/P70832</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-004514-ladsgroup.json</li> <li>00:35 ladsgroup@cumin1002: dbctl commit (dc=all): 'Depooling db2181 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70831">https://phabricator.wikimedia.org/P70831</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-003546-ladsgroup.json</li> <li>00:35 ladsgroup@cumin1002: END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2181.codfw.wmnet with reason: Maintenance</li> <li>00:35 ladsgroup@cumin1002: START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2181.codfw.wmnet with reason: Maintenance</li> <li>00:35 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2167 (<a href="https://phabricator.wikimedia.org/T376905" class="extiw" title="phab:T376905">T376905</a>)', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70830">https://phabricator.wikimedia.org/P70830</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-003520-ladsgroup.json</li> <li>00:20 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2167', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70829">https://phabricator.wikimedia.org/P70829</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-002013-ladsgroup.json</li> <li>00:05 ladsgroup@cumin1002: dbctl commit (dc=all): 'Repooling after maintenance db2167', diff saved to <a class="external free" href="https://phabricator.wikimedia.org/P70828">https://phabricator.wikimedia.org/P70828</a> and previous config saved to /var/cache/conftool/dbconfig/20241101-000506-ladsgroup.json</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Archives" data-mw-thread-id="h-Archives"><span data-mw-comment-start="" id="h-Archives"></span>Archives<span data-mw-comment-end="h-Archives"></span></h2></div> <p>See <a href="/wiki/Server_Admin_Log/Archives" title="Server Admin Log/Archives">Server Admin Log/Archives</a>. </p>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://wikitech.wikimedia.org/w/index.php?title=Server_Admin_Log&oldid=2247060">https://wikitech.wikimedia.org/w/index.php?title=Server_Admin_Log&oldid=2247060</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Special:Categories" title="Special:Categories">Categories</a>: <ul><li><a href="/wiki/Category:SAL" title="Category:SAL">SAL</a></li><li><a href="/wiki/Category:Operations" title="Category:Operations">Operations</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 23 November 2024, at 12:08.</li> <li id="footer-info-copyright">Text is available under the <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-ShareAlike License</a>; additional terms may apply. See <a class="external text" href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use">Terms of Use</a> for details.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Main_Page">About Wikitech</a></li> <li id="footer-places-disclaimers"><a href="https://foundation.wikimedia.org/wiki/General_disclaimer">Disclaimers</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/wikitech.wikimedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//wikitech.wikimedia.org/w/index.php?title=Server_Admin_Log&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-f69cdc8f6-vthpd","wgBackendResponseTime":112,"wgDiscussionToolsPageThreads":[{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-23","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-22","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-21","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-20","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-19","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-18","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-17","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-16","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-15","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-14","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-13","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-12","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-11","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-10","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-09","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-08","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-07","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-06","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-05","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-04","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-02","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-2024-11-01","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Archives","replies":[]}],"wgPageParseReport":{"discussiontools":{"limitreport-timeusage":"0.189"},"limitreport":{"cputime":"0.570","walltime":"0.573","ppvisitednodes":{"value":2641,"limit":1000000},"postexpandincludesize":{"value":19599,"limit":2097152},"templateargumentsize":{"value":377,"limit":2097152},"expansiondepth":{"value":5,"limit":100},"expensivefunctioncount":{"value":0,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":44,"limit":5000000},"timingprofile":["100.00% 52.145 1 -total"," 85.58% 44.625 48 Template:Gerrit"]},"scribunto":{"limitreport-timeusage":{"value":"0.022","limit":"10.000"},"limitreport-memusage":{"value":885791,"limit":52428800}},"cachereport":{"origin":"mw-api-ext.codfw.main-7556f8b5dd-ttfjp","timestamp":"20241123120859","ttl":2592000,"transientcontent":false}}});});</script> </body> </html>