CINXE.COM

Data Platform/Systems/Managing systemd timers - Wikitech

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>Data Platform/Systems/Managing systemd timers - Wikitech</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )labswikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat": "dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"0f968793-ac90-4e42-a470-eacfaef6471e","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Data_Platform/Systems/Managing_systemd_timers","wgTitle":"Data Platform/Systems/Managing systemd timers","wgCurRevisionId":2201923,"wgRevisionId":2201923,"wgArticleId":443309,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Data platform","Data platform systems"],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Data_Platform/Systems/Managing_systemd_timers","wgRelevantArticleId":443309,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikitech","wgCiteReferencePreviewsActive":true, "wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":10000,"wgCentralAuthMobileDomain":false,"wgEditSubmitButtonLabelPublish":true,"wgDiscussionToolsFeaturesEnabled":{"replytool":true,"newtopictool":true,"sourcemodetoolbar":true,"topicsubscription":false,"autotopicsub":false,"visualenhancements":false,"visualenhancements_reply":false,"visualenhancements_pageframe":false},"wgDiscussionToolsFallbackEditMode":"visual","wgULSPosition":"personal","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"wgSiteNoticeId":"2.0"};RLSTATE={ "ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.pygments":"ready","ext.discussionTools.init.styles":"ready","oojs-ui-core.styles":"ready","oojs-ui.styles.indicators":"ready","mediawiki.widgets.styles":"ready","oojs-ui-core.icons":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.pt":"ready","ext.dismissableSiteNotice.styles":"ready"};RLPAGEMODULES=["ext.pygments.view","site","mediawiki.page.ready","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.site","ext.urlShortener.toolbar","ext.centralauth.centralautologin","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.discussionTools.init","ext.eventLogging", "ext.wikimediaEvents","ext.uls.interface","ext.checkUser.clientHints","ext.dismissableSiteNotice"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&amp;modules=ext.discussionTools.init.styles%7Cext.dismissableSiteNotice.styles%7Cext.pygments%7Cext.uls.pt%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cmediawiki.widgets.styles%7Coojs-ui-core.icons%2Cstyles%7Coojs-ui.styles.indicators%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles&amp;only=styles&amp;skin=vector-2022"> <script async="" src="/w/load.php?lang=en&amp;modules=startup&amp;only=scripts&amp;raw=1&amp;skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&amp;modules=site.styles&amp;only=styles&amp;skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.4"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="Data Platform/Systems/Managing systemd timers - Wikitech"> <meta property="og:type" content="website"> <link rel="icon" href="/static/favicon/wikitech.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikitech (en)"> <link rel="EditURI" type="application/rsd+xml" href="//wikitech.wikimedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://wikitech.wikimedia.org/wiki/Data_Platform/Systems/Managing_systemd_timers"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/"> <link rel="alternate" type="application/atom+xml" title="Wikitech Atom feed" href="/w/index.php?title=Special:RecentChanges&amp;feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="ext-discussiontools-replytool-enabled ext-discussiontools-newtopictool-enabled ext-discussiontools-sourcemodetoolbar-enabled skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Data_Platform_Systems_Managing_systemd_timers rootpage-Data_Platform skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-Server-admin-log:-Prod" class="mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Server admin log: Prod</span></a></li><li id="n-Admin-log:-RelEng" class="mw-list-item"><a href="/wiki/Release_Engineering/SAL"><span>Admin log: RelEng</span></a></li><li id="n-Incident-status" class="mw-list-item"><a href="/wiki/Incident_status"><span>Incident status</span></a></li><li id="n-Deployments" class="mw-list-item"><a href="/wiki/Deployments"><span>Deployments</span></a></li><li id="n-SRE-Team-Help" class="mw-list-item"><a href="/wiki/SRE/SRE_Team_requests"><span>SRE Team Help</span></a></li> </ul> </div> </div> <div id="p-Cloud_VPS_&amp;_Toolforge" class="vector-menu mw-portlet mw-portlet-Cloud_VPS_Toolforge" > <div class="vector-menu-heading"> Cloud VPS &amp; Toolforge </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-Cloud-VPS-portal" class="mw-list-item"><a href="/wiki/Portal:Cloud_VPS"><span>Cloud VPS portal</span></a></li><li id="n-Toolforge-portal" class="mw-list-item"><a href="/wiki/Portal:Toolforge"><span>Toolforge portal</span></a></li><li id="n-Request-VPS-project" class="mw-list-item"><a href="https://phabricator.wikimedia.org/project/view/2875/"><span>Request VPS project</span></a></li><li id="n-Admin-log:-Cloud-VPS" class="mw-list-item"><a href="/wiki/Cloud_VPS_Server_Admin_Log"><span>Admin log: Cloud VPS</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikitech.svg" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikitech" src="/static/images/mobile/copyright/wikitech-wordmark.svg" style="width: 8.75em; height: 1.6875em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikitech [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikitech" aria-label="Search Wikitech" autocapitalize="sentences" title="Search Wikitech [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-uls" class="mw-list-item active user-links-collapsible-item"><a data-mw="interface" href="#" class="uls-trigger cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>English</span></a> </li> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page&#039;s font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?utm_source=donate&amp;utm_medium=sidebar&amp;utm_campaign=spontaneous&amp;uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&amp;returnto=Data+Platform%2FSystems%2FManaging+systemd+timers" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out user-links-collapsible-item" title="More options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?utm_source=donate&amp;utm_medium=sidebar&amp;utm_campaign=spontaneous&amp;uselang=en"><span>Donate</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&amp;returnto=Data+Platform%2FSystems%2FManaging+systemd+timers" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"><div id="mw-dismissablenotice-anonplace"></div><script>(function(){var node=document.getElementById("mw-dismissablenotice-anonplace");if(node){node.outerHTML="\u003Cdiv class=\"mw-dismissable-notice\"\u003E\u003Cdiv class=\"mw-dismissable-notice-close\"\u003E[\u003Ca tabindex=\"0\" role=\"button\"\u003Edismiss\u003C/a\u003E]\u003C/div\u003E\u003Cdiv class=\"mw-dismissable-notice-body\"\u003E\u003C!-- CentralNotice --\u003E\u003Cdiv id=\"localNotice\" data-nosnippet=\"\"\u003E\u003Cdiv class=\"sitenotice\" lang=\"en\" dir=\"ltr\"\u003E\u003Ctable style=\"width: 75%; background-color: var(--background-color-warning-subtle, #fdf2d5); border: var(--border-subtle, 1px solid #987027); color: var(--color-base, #202122); border-radius: 10px; padding: 5px; margin: 0 auto;\"\u003E\n\u003Ctbody\u003E\u003Ctr\u003E\n\u003Ctd style=\"width:40px; height:40px; text-align:center; vertical-align:middle; padding: 2px;\"\u003E\u003Cspan typeof=\"mw:File\"\u003E\u003Ca href=\"/wiki/File:OOjs_UI_icon_alert-warning.svg\" class=\"mw-file-description\"\u003E\u003Cimg src=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/30px-OOjs_UI_icon_alert-warning.svg.png\" decoding=\"async\" width=\"30\" height=\"30\" class=\"mw-file-element\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/45px-OOjs_UI_icon_alert-warning.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/60px-OOjs_UI_icon_alert-warning.svg.png 2x\" data-file-width=\"20\" data-file-height=\"20\" /\u003E\u003C/a\u003E\u003C/span\u003E\n\u003C/td\u003E\n\u003Ctd style=\"text-align:center; vertical-align: middle; padding: 4px; max-height: 60px;\"\u003E\u003Cb\u003EWe are migrating Wikitech to \u003Ca href=\"/wiki/Wikitech/SUL-migration\" title=\"Wikitech/SUL-migration\"\u003ESUL\u003C/a\u003E!\u003C/b\u003E\n\u003Cp\u003E\u003Cb\u003EAction may be required for your \u003Ca href=\"/wiki/Wikitech/SUL-migration#What_You_Should_Do\" title=\"Wikitech/SUL-migration\"\u003E account\u003C/a\u003E!\u003C/b\u003E\n\u003C/p\u003E\u003Cp\u003E\u003Cb\u003ETrouble logging in? Please visit \u003Ca href=\"https://phabricator.wikimedia.org/T376267\" class=\"extiw\" title=\"phab:T376267\"\u003ET376267\u003C/a\u003E\u003C/b\u003E\n\u003C/p\u003E\n\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E";}}());</script></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">Beginning</div> </a> </li> <li id="toc-Anatomy_of_a_Systemd_timer" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Anatomy_of_a_Systemd_timer"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Anatomy of a Systemd timer</span> </div> </a> <button aria-controls="toc-Anatomy_of_a_Systemd_timer-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Anatomy of a Systemd timer subsection</span> </button> <ul id="toc-Anatomy_of_a_Systemd_timer-sublist" class="vector-toc-list"> <li id="toc-ExecStart_and_difference_with_bash" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#ExecStart_and_difference_with_bash"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.1</span> <span>ExecStart and difference with bash</span> </div> </a> <ul id="toc-ExecStart_and_difference_with_bash-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Calculating_OnCalendar_interval" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Calculating_OnCalendar_interval"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.2</span> <span>Calculating OnCalendar interval</span> </div> </a> <ul id="toc-Calculating_OnCalendar_interval-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Operations" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Operations"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Operations</span> </div> </a> <button aria-controls="toc-Operations-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Operations subsection</span> </button> <ul id="toc-Operations-sublist" class="vector-toc-list"> <li id="toc-List_all_timers_running_on_the_host" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#List_all_timers_running_on_the_host"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1</span> <span>List all timers running on the host</span> </div> </a> <ul id="toc-List_all_timers_running_on_the_host-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-List_command_timers_are_actually_running" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#List_command_timers_are_actually_running"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.2</span> <span>List command timers are actually running</span> </div> </a> <ul id="toc-List_command_timers_are_actually_running-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Check_which_user_runs_a_timer" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Check_which_user_runs_a_timer"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.3</span> <span>Check which user runs a timer</span> </div> </a> <ul id="toc-Check_which_user_runs_a_timer-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Disable_a_Systemd_timer" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Disable_a_Systemd_timer"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.4</span> <span>Disable a Systemd timer</span> </div> </a> <ul id="toc-Disable_a_Systemd_timer-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Shut_off_/_Reset_Icinga_alert_already_fired" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Shut_off_/_Reset_Icinga_alert_already_fired"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.5</span> <span>Shut off / Reset Icinga alert already fired</span> </div> </a> <ul id="toc-Shut_off_/_Reset_Icinga_alert_already_fired-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Check_logs_for_a_systemd_timer" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Check_logs_for_a_systemd_timer"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.6</span> <span>Check logs for a systemd timer</span> </div> </a> <ul id="toc-Check_logs_for_a_systemd_timer-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Failed_service_procedure_(ops_week)" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Failed_service_procedure_(ops_week)"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Failed service procedure (ops week)</span> </div> </a> <button aria-controls="toc-Failed_service_procedure_(ops_week)-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Failed service procedure (ops week) subsection</span> </button> <ul id="toc-Failed_service_procedure_(ops_week)-sublist" class="vector-toc-list"> <li id="toc-Checking_logs" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Checking_logs"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Checking logs</span> </div> </a> <ul id="toc-Checking_logs-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Resetting_service_status_to_normal" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Resetting_service_status_to_normal"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.2</span> <span>Resetting service status to normal</span> </div> </a> <ul id="toc-Resetting_service_status_to_normal-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Data Platform/Systems/Managing systemd timers</span></h1> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Data_Platform/Systems/Managing_systemd_timers" title="View the content page [c]" accesskey="c"><span>Page</span></a></li><li id="ca-talk" class="new vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Talk:Data_Platform/Systems/Managing_systemd_timers&amp;action=edit&amp;redlink=1" rel="discussion" class="new" title="Discussion about the content page (page does not exist) [t]" accesskey="t"><span>Discussion</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/Data_Platform/Systems/Managing_systemd_timers"><span>Read</span></a></li><li id="ca-viewsource" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;action=edit" title="This page is protected.&#10;You can view its source [e]" accesskey="e"><span>View source</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/Data_Platform/Systems/Managing_systemd_timers"><span>Read</span></a></li><li id="ca-more-viewsource" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;action=edit"><span>View source</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/Data_Platform/Systems/Managing_systemd_timers" title="A list of all wiki pages that link here [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/Data_Platform/Systems/Managing_systemd_timers" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;oldid=2201923" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&amp;page=Data_Platform%2FSystems%2FManaging_systemd_timers&amp;id=2201923&amp;wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&amp;url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FData_Platform%2FSystems%2FManaging_systemd_timers"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&amp;url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FData_Platform%2FSystems%2FManaging_systemd_timers"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-create_a_book" class="mw-list-item"><a href="/w/index.php?title=Special:Book&amp;bookcmd=book_creator&amp;referer=Data+Platform%2FSystems%2FManaging+systemd+timers"><span>Create a book</span></a></li><li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&amp;page=Data_Platform%2FSystems%2FManaging_systemd_timers&amp;action=show-download-screen"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikitech</div> </div> <div id="contentSub"><div id="mw-content-subtitle"><div class="subpages">&lt; <bdi dir="ltr"><a href="/wiki/Data_Platform" title="Data Platform">Data Platform</a></bdi> | <bdi dir="ltr"><a href="/wiki/Data_Platform/Systems" title="Data Platform/Systems">Systems</a></bdi></div></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><p>In <a href="https://phabricator.wikimedia.org/T172532" class="extiw" title="phab:T172532">phab:T172532</a> we decided to move away as much as possible from cron jobs since we were struggling to alarm when a job failed. Most of our recurrent jobs are running on an-coord1001 (analytics cluster coordinator), and the majority of them are systemd timers.<style data-mw-deduplicate="TemplateStyles:r2211903">.mw-parser-output .note{background-position:left 7px top 50%;padding:0.5em 0.5em 0.5em 40px;margin:0.5em 0;overflow:hidden;background-color:#f8f9fa;color:#333;background-repeat:no-repeat;border:1px solid #ddd}.mw-parser-output .note-inline{display:inline-block;vertical-align:middle}.mw-parser-output .note-info{background-color:#eaf3ff;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg");background-size:25px;border-color:#a3caff;padding-left:40px;min-height:25px}.mw-parser-output .note-reminder{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/a/a8/OOjs_UI_icon_lightbulb-yellow.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-warn{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/3/3b/OOjs_UI_icon_alert-warning.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-error{background-color:#fee7e6;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/b/bf/OOjs_UI_icon_notice-destructive.svg");background-size:25px;border-color:#c33;min-height:25px}@media screen{html.skin-theme-clientpref-night .mw-parser-output .note{background-color:transparent;color:inherit}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .note{background-color:transparent;color:inherit}}</style></p><div role="note" class="note note-warn">Timers are different from cron-jobs noticeably in that they automatically restart failed jobs. If you need a timer to be stopped and executed later, first disable the timer (see section below), then kill the currently running job, and finally re-enable the timer at will.</div> <meta property="mw:PageProp/toc"/> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Anatomy_of_a_Systemd_timer" data-mw-thread-id="h-Anatomy_of_a_Systemd_timer"><span data-mw-comment-start="" id="h-Anatomy_of_a_Systemd_timer"></span>Anatomy of a Systemd timer<span data-mw-comment-end="h-Anatomy_of_a_Systemd_timer"></span></h2><!--__DTELLIPSISBUTTON__{"threadItem":{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Anatomy_of_a_Systemd_timer","replies":["h-ExecStart_and_difference_with_bash-Anatomy_of_a_Systemd_timer","h-Calculating_OnCalendar_interval-Anatomy_of_a_Systemd_timer"]}}--></div><p> A systemd timer is composed by two parts: a unit and the timer itself. Let's pick an example:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>elukey@an-coord1001:~$<span class="w"> </span>sudo<span class="w"> </span>systemctl<span class="w"> </span>cat<span class="w"> </span>hdfs-balancer.timer <span class="c1"># /lib/systemd/system/hdfs-balancer.timer</span> <span class="o">[</span>Unit<span class="o">]</span> <span class="nv">Description</span><span class="o">=</span>Periodic<span class="w"> </span>execution<span class="w"> </span>of<span class="w"> </span>hdfs-balancer.service <span class="o">[</span>Timer<span class="o">]</span> <span class="nv">Unit</span><span class="o">=</span>hdfs-balancer.service <span class="c1"># Accuracy sets the maximum time interval around the execution time we want to allow</span> <span class="nv">AccuracySec</span><span class="o">=</span>15sec <span class="nv">OnCalendar</span><span class="o">=</span>*-*-*<span class="w"> </span><span class="m">06</span>:00:00 <span class="nv">RandomizedDelaySec</span><span class="o">=</span><span class="m">0</span> <span class="o">[</span>Install<span class="o">]</span> <span class="nv">WantedBy</span><span class="o">=</span>multi-user.target elukey@an-coord1001:~$<span class="w"> </span>sudo<span class="w"> </span>systemctl<span class="w"> </span>cat<span class="w"> </span>hdfs-balancer.service <span class="c1"># /lib/systemd/system/hdfs-balancer.service</span> <span class="o">[</span>Unit<span class="o">]</span> <span class="nv">Description</span><span class="o">=</span>Run<span class="w"> </span>the<span class="w"> </span>HDFS<span class="w"> </span>balancer<span class="w"> </span>script<span class="w"> </span>to<span class="w"> </span>keep<span class="w"> </span>HDFS<span class="w"> </span>blocks<span class="w"> </span>replicated<span class="w"> </span><span class="k">in</span><span class="w"> </span>the<span class="w"> </span>most<span class="w"> </span>redundant<span class="w"> </span>and<span class="w"> </span>efficient<span class="w"> </span>way. <span class="o">[</span>Service<span class="o">]</span> <span class="nv">User</span><span class="o">=</span>hdfs <span class="nv">ExecStart</span><span class="o">=</span>/usr/local/bin/hdfs-balancer </pre></div><p>In this case, we are checking how it is configured the HDFS balancer timer. The .timer file references a systemd unit (the .service file), that it is the one doing the work, and a pattern of execution (in this case, every day at 6 AM UTC). The .service file defines a command, together with other info like environment variables and username that needs to execute it. This command that .service points to can be generated by puppet, so peek into it if you need to see the details, for example: <code>sudo -u hdfs cat /usr/local/bin/refinery-sqoop-mediawiki</code>. </p><p>We alarm on the return code of the .service Systemd unit: if it gets executed and the script mentioned by <b>ExecStart</b> returns a non-zero exit code, Nagios will alert us. </p><p> All these configuration files seem a bit confusing, but they are good from the operations point of view. A systemd .service can be started/restarted any time, basically forcing the execution of its script. In this case:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>start<span class="w"> </span>hdfs-balancer.service </pre></div><p>forces the hdfs-balancer to execute. Where do we find logs? At the moment the majority of them are only on journald, but they will be logged on files as well as soon as possible. If you want to check journald for a particular .service:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span><span class="c1"># start from the first log registered</span> sudo<span class="w"> </span>journalctl<span class="w"> </span>-u<span class="w"> </span>hdfs-balancer <span class="c1"># start from a specific moment of time</span> sudo<span class="w"> </span>journalctl<span class="w"> </span>-u<span class="w"> </span>hdfs-balancer<span class="w"> </span>--since<span class="w"> </span><span class="s1">'3 days ago'</span> <span class="c1"># show only a specific timeframe</span> sudo<span class="w"> </span>journalctl<span class="w"> </span>--since<span class="w"> </span><span class="s2">"2018-10-21 23:15:00"</span><span class="w"> </span>--until<span class="w"> </span><span class="s2">"2018-10-21 23:20:00"</span> <span class="c1"># tail -f equivalent</span> sudo<span class="w"> </span>journalctl<span class="w"> </span>-u<span class="w"> </span>hdfs-balancer<span class="w"> </span>--follow </pre></div><p>An example of debugging for db1107: <a class="external free" href="https://phabricator.wikimedia.org/T207165">https://phabricator.wikimedia.org/T207165</a> </p><div class="mw-heading mw-heading3"><h3 id="ExecStart_and_difference_with_bash" data-mw-thread-id="h-ExecStart_and_difference_with_bash-Anatomy_of_a_Systemd_timer"><span data-mw-comment-start="" id="h-ExecStart_and_difference_with_bash-Anatomy_of_a_Systemd_timer"></span>ExecStart and difference with bash<span data-mw-comment-end="h-ExecStart_and_difference_with_bash-Anatomy_of_a_Systemd_timer"></span></h3></div> <p>Please note that the command executed by the service unit (the one assigned to the ExecStart field) is parsed following a systemd specific format, that is similar to what bash offers but very different in may ways. If you don't keep in mind this rule you'll end up writing units that either fail due to weird argument parsing or even worse executing with wrong parameters. The fix in <a class="external free" href="https://gerrit.wikimedia.org/r/#/c/operations/puppet/+/530555/">https://gerrit.wikimedia.org/r/#/c/operations/puppet/+/530555/</a> is a good example: a shell redirection ( command > file.log) is not parsed as we expect by systemd, that 1) considers '>' as special char and 2) considers everything split by a whitespace (and not quoted) as separate argument. </p><p><br/> </p> <div class="mw-heading mw-heading3"><h3 id="Calculating_OnCalendar_interval" data-mw-thread-id="h-Calculating_OnCalendar_interval-Anatomy_of_a_Systemd_timer"><span data-mw-comment-start="" id="h-Calculating_OnCalendar_interval-Anatomy_of_a_Systemd_timer"></span>Calculating OnCalendar interval<span data-mw-comment-end="h-Calculating_OnCalendar_interval-Anatomy_of_a_Systemd_timer"></span></h3></div> <p>The OnCalender interval syntax is powerful, but isn't always the most straightforward to figure out. There is a command to help: <code>systemd-analyze</code>. </p><p>To check the syntax of OnCalendar, you can run e.g. </p> <div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>$<span class="w"> </span>systemd-analyze<span class="w"> </span>calendar<span class="w"> </span><span class="s1">'*-*-* *:05,35:00'</span> Normalized<span class="w"> </span>form:<span class="w"> </span>*-*-*<span class="w"> </span>*:05,35:00 <span class="w"> </span>Next<span class="w"> </span>elapse:<span class="w"> </span>Thu<span class="w"> </span><span class="m">2021</span>-07-08<span class="w"> </span><span class="m">14</span>:35:00<span class="w"> </span>UTC <span class="w"> </span>From<span class="w"> </span>now:<span class="w"> </span>24min<span class="w"> </span>left </pre></div> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Operations" data-mw-thread-id="h-Operations"><span data-mw-comment-start="" id="h-Operations"></span>Operations<span data-mw-comment-end="h-Operations"></span></h2><!--__DTELLIPSISBUTTON__{"threadItem":{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Operations","replies":["h-List_all_timers_running_on_the_host-Operations","h-List_command_timers_are_actually_running-Operations","h-Check_which_user_runs_a_timer-Operations","h-Disable_a_Systemd_timer-Operations","h-Shut_off_\/_Reset_Icinga_alert_already_fired-Operations","h-Check_logs_for_a_systemd_timer-Operations"]}}--></div> <div class="mw-heading mw-heading3"><h3 id="List_all_timers_running_on_the_host" data-mw-thread-id="h-List_all_timers_running_on_the_host-Operations"><span data-mw-comment-start="" id="h-List_all_timers_running_on_the_host-Operations"></span>List all timers running on the host<span data-mw-comment-end="h-List_all_timers_running_on_the_host-Operations"></span></h3></div> <div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>systemctl<span class="w"> </span>list-timers </pre></div><p>Example of output from stat1006:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>elukey@stat1006:~$<span class="w"> </span>systemctl<span class="w"> </span>list-timers NEXT<span class="w"> </span>LEFT<span class="w"> </span>LAST<span class="w"> </span>PASSED<span class="w"> </span>UNIT<span class="w"> </span>ACTIVATES Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:43:00<span class="w"> </span>UTC<span class="w"> </span>2min<span class="w"> </span>54s<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:28:00<span class="w"> </span>UTC<span class="w"> </span>12min<span class="w"> </span>ago<span class="w"> </span>prometheus-node-exporter-apt.timer<span class="w"> </span>prometheus-node-exporter-apt.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-cx.timer<span class="w"> </span>reportupdater-cx.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-edit-beta-features.timer<span class="w"> </span>reportupdater-edit-beta-features.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-ee-beta-features.timer<span class="w"> </span>reportupdater-ee-beta-features.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-ee.timer<span class="w"> </span>reportupdater-ee.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-flow-beta-features.timer<span class="w"> </span>reportupdater-flow-beta-features.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-flow.timer<span class="w"> </span>reportupdater-flow.service Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">07</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>19min<span class="w"> </span>left<span class="w"> </span>Wed<span class="w"> </span><span class="m">2019</span>-03-27<span class="w"> </span><span class="m">06</span>:00:00<span class="w"> </span>UTC<span class="w"> </span>40min<span class="w"> </span>ago<span class="w"> </span>reportupdater-language.timer<span class="w"> </span>reportupdater-language.service </pre></div> <p>To check on a specific timer, use status: <code>systemctl status refinery-sqoop-whole-mediawiki</code> </p><p>It is easy to see all the timings for a certain systemd timer and the units it runs. For example, the <code>reportupdater-cx</code> job has two units: </p> <ul><li><code>reportupdater-cx.timer</code> -> this is basically the equivalent of the cron definition (you can inspect it via <code>systemctl cat reportupdater-cx.timer</code> and see its config)</li> <li><code>reportupdater-cx.service</code> -> this is the command to run together with its environment etc.., that is called/managed by the above <code>.timer</code> unit.</li></ul> <div class="mw-heading mw-heading3"><h3 id="List_command_timers_are_actually_running" data-mw-thread-id="h-List_command_timers_are_actually_running-Operations"><span data-mw-comment-start="" id="h-List_command_timers_are_actually_running-Operations"></span>List command timers are actually running<span data-mw-comment-end="h-List_command_timers_are_actually_running-Operations"></span></h3></div> <div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>nuria@an-launcher1002:~$<span class="w"> </span>systemctl<span class="w"> </span>cat<span class="w"> </span>drop-el-unsanitized-events.service <span class="c1"># /lib/systemd/system/drop-el-unsanitized-events.service</span> <span class="o">[</span>Unit<span class="o">]</span> <span class="nv">Description</span><span class="o">=</span>Drop<span class="w"> </span>unsanitized<span class="w"> </span>EventLogging<span class="w"> </span>data<span class="w"> </span>from<span class="w"> </span>the<span class="w"> </span>event<span class="w"> </span>database<span class="w"> </span>after<span class="w"> </span>retention<span class="w"> </span>period. <span class="o">[</span>Service<span class="o">]</span> <span class="nv">User</span><span class="o">=</span>analytics <span class="nv">Environment</span><span class="o">=</span><span class="nv">PYTHONPATH</span><span class="o">=</span><span class="si">${</span><span class="nv">PYTHONPATH</span><span class="si">}</span>:/srv/deployment/analytics/refinery/python <span class="nv">ExecStart</span><span class="o">=</span>/usr/local/bin/kerberos-run-command<span class="w"> </span>analytics<span class="w"> </span>/srv/deployment/analytics/refinery/bin/refinery-drop-older-than<span class="w"> </span>--some </pre></div> <div class="mw-heading mw-heading3"><h3 id="Check_which_user_runs_a_timer" data-mw-thread-id="h-Check_which_user_runs_a_timer-Operations"><span data-mw-comment-start="" id="h-Check_which_user_runs_a_timer-Operations"></span>Check which user runs a timer<span data-mw-comment-end="h-Check_which_user_runs_a_timer-Operations"></span></h3></div><p> Everything always starts with a <code>systemctl list-timers</code> (as explained above) to find the .timer unit to check. If we keep the stat1006 example above, let's say we want to check what user runs <code>reportupdater-cx.timer</code>: we'll need to inspect the related .service unit (since that one is the "real" command executed, the .timer one is only the cron-like definition and we don't care who runs it):</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>elukey@stat1006:~$<span class="w"> </span>systemctl<span class="w"> </span>cat<span class="w"> </span>reportupdater-cx <span class="c1"># /lib/systemd/system/reportupdater-cx.service</span> <span class="o">[</span>Unit<span class="o">]</span> <span class="nv">Description</span><span class="o">=</span>Report<span class="w"> </span>Updater<span class="w"> </span>job<span class="w"> </span><span class="k">for</span><span class="w"> </span>cx <span class="o">[</span>Service<span class="o">]</span> <span class="nv">User</span><span class="o">=</span>stats <span class="nv">SyslogIdentifier</span><span class="o">=</span>reportupdater-cx <span class="nv">ExecStart</span><span class="o">=</span>/usr/bin/python<span class="w"> </span>/srv/reportupdater/reportupdater/update_reports.py<span class="w"> </span>-l<span class="w"> </span>info<span class="w"> </span>/srv/reportupdater/jobs/limn-language-data/cx<span class="w"> </span>/srv/reportupdater/output/metrics/cx </pre></div><p>You need to look for the <code>User=something</code> definition. In this case, the unit is run by <code>stats</code>. </p><div class="mw-heading mw-heading3"><h3 id="Disable_a_Systemd_timer" data-mw-thread-id="h-Disable_a_Systemd_timer-Operations"><span data-mw-comment-start="" id="h-Disable_a_Systemd_timer-Operations"></span>Disable a Systemd timer<span data-mw-comment-end="h-Disable_a_Systemd_timer-Operations"></span></h3></div> <link rel="mw-deduplicated-inline-style" href="mw-data:TemplateStyles:r2211903"><div role="note" class="note note-warn">Very important: to keep the timer disabled/stopped puppet needs to be disabled for the time being.</div><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>disable<span class="w"> </span>name-of-the-timer.timer sudo<span class="w"> </span>systemctl<span class="w"> </span>stop<span class="w"> </span>name-of-the-timer.timer </pre></div><p>The former prevents the timer to be started after boot, the latter removes it from the regular scheduling. After you have done the above steps, verify with the following command that the timer is not listed anymore:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>systemctl<span class="w"> </span>list-timers<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>name-of-the-timer </pre></div><p>Finally don't forget to <b>check if an icinga alarm was fired</b>, and follow the next section accordingly if needed. </p><div class="mw-heading mw-heading3"><h3 id="Shut_off_/_Reset_Icinga_alert_already_fired" data-mw-thread-id="h-Shut_off_/_Reset_Icinga_alert_already_fired-Operations"><span id="Shut_off_.2F_Reset_Icinga_alert_already_fired"></span><span data-mw-comment-start="" id="h-Shut_off_/_Reset_Icinga_alert_already_fired-Operations"></span>Shut off / Reset Icinga alert already fired<span data-mw-comment-end="h-Shut_off_/_Reset_Icinga_alert_already_fired-Operations"></span></h3></div><p> It might happen that an alert has fired for a timer but you just want to shut it off, because you have already been taken care of the problem (for example, executing a manual run, etc..). This is what is needed to do:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>reset-failed<span class="w"> </span>name-of-the-timer.service </pre></div><p>It is important to reset the service unit, not the timer one, since it is only the former that causes the alarm (the timer unit's state is not important at this stage). </p><p>In the case of timers that run multiple commands, like refinery-sqoop-whole-mediawiki, see if you can finish running the commands manually and reset the alert as above. There's nothing special about these timers that can help resume operation or anything like that. </p> <div class="mw-heading mw-heading3"><h3 id="Check_logs_for_a_systemd_timer" data-mw-thread-id="h-Check_logs_for_a_systemd_timer-Operations"><span data-mw-comment-start="" id="h-Check_logs_for_a_systemd_timer-Operations"></span>Check logs for a systemd timer<span data-mw-comment-end="h-Check_logs_for_a_systemd_timer-Operations"></span></h3></div><p> The first step is always to use systemctl list-timers, as explained above, to gather the name of the timer to check. In the above stat1006 example, let's pick <code>reportupdater-cx.timer</code>. The logs will be available simply doing:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span><span class="c1"># Note that we are not using the .timer unit, but the .service one!</span> <span class="c1"># The .service suffix is assumed if no one is specified.</span> sudo<span class="w"> </span>journalctl<span class="w"> </span>-u<span class="w"> </span>reportupdater-cx </pre></div><p>Usually for important jobs we log stuff to disk, since journald by default logs to a tmpfs partition (that stays in memory and gets flushed on each reboot). In order to see what is the log configuration, two things can be done: </p><ul><li>Check the puppet definition of the systemd timer, it will list the log directory.</li> <li>Check rsyslog.d configuration on the host itself.</li></ul><p> Example for the former:</p><div class="mw-highlight mw-highlight-lang-puppet mw-content-ltr" dir="ltr"><pre><span></span><span class="w"> </span><span class="na">systemd</span><span class="p">::</span><span class="na">timer</span><span class="p">::</span><span class="na">job</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="s">"reportupdater-${title}"</span><span class="p">:</span> <span class="w"> </span><span class="na">ensure</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$ensure,</span> <span class="w"> </span><span class="na">description</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">"Report Updater job for ${title}"</span><span class="p">,</span> <span class="w"> </span><span class="na">command</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">"/usr/bin/python ${::reportupdater::path}/update_reports.py -l info ${query_path} ${output_path}"</span><span class="p">,</span> <span class="w"> </span><span class="na">interval</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="p">{</span> <span class="w"> </span><span class="s">'start'</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">'OnCalendar'</span><span class="p">,</span> <span class="w"> </span><span class="s">'interval'</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$interval</span> <span class="w"> </span><span class="p">},</span> <span class="w"> </span><span class="k">user</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$::reportupdater::user,</span> <span class="w"> </span><span class="na">monitoring_enabled</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$monitoring_enabled,</span> <span class="w"> </span><span class="na">monitoring_contact_groups</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">'analytics'</span><span class="p">,</span> <span class="w"> </span><span class="na">logging_enabled</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="k">true</span><span class="p">,</span> <span class="w"> </span><span class="na">logfile_basedir</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$::reportupdater::log_path,</span><span class="w"> </span><span class="o">&lt;==================</span> <span class="w"> </span><span class="na">logfile_name</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">'syslog.log'</span><span class="p">,</span> <span class="w"> </span><span class="na">logfile_owner</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$::reportupdater::user,</span> <span class="w"> </span><span class="na">logfile_group</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="nv">$::reportupdater::user,</span> <span class="w"> </span><span class="na">logfile_perms</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">'all'</span><span class="p">,</span> <span class="w"> </span><span class="na">syslog_force_stop</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="k">true</span><span class="p">,</span> <span class="w"> </span><span class="na">syslog_identifier</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="s">"reportupdater-${title}"</span><span class="p">,</span> <span class="w"> </span><span class="p">}</span> </pre></div><p>About the latter: why rsyslog? Wasn't it journald? We have a rsyslog hooked in into journald so we can add specific rules to create log files. For example, let's check the <code>reportupdater-cx</code> job:</p><div class="mw-highlight mw-highlight-lang-bash mw-content-ltr" dir="ltr"><pre><span></span>elukey@stat1006:~$<span class="w"> </span>cat<span class="w"> </span>/etc/rsyslog.d/ <span class="m">10</span>-exporter.conf<span class="w"> </span><span class="m">20</span>-reportupdater-edit-beta-features.conf<span class="w"> </span><span class="m">20</span>-reportupdater-flow-beta-features.conf<span class="w"> </span><span class="m">20</span>-reportupdater-mt-engines.conf<span class="w"> </span><span class="m">20</span>-reportupdater-published-cx2-translations.conf <span class="m">10</span>-puppet-agent.conf<span class="w"> </span><span class="m">20</span>-reportupdater-ee-beta-features.conf<span class="w"> </span><span class="m">20</span>-reportupdater-flow.conf<span class="w"> </span><span class="m">20</span>-reportupdater-page-creation.conf<span class="w"> </span><span class="m">30</span>-remote-syslog.conf <span class="m">20</span>-reportupdater-cx.conf<span class="w"> </span><span class="m">20</span>-reportupdater-ee.conf<span class="w"> </span><span class="m">20</span>-reportupdater-language.conf<span class="w"> </span><span class="m">20</span>-reportupdater-pingback.conf<span class="w"> </span>README elukey@stat1006:~$<span class="w"> </span>cat<span class="w"> </span>/etc/rsyslog.d/20-reportupdater-cx.conf <span class="c1"># rsyslogd(8) configuration file for services.</span> <span class="c1"># This file is managed by Puppet.</span> :programname,<span class="w"> </span>startswith,<span class="w"> </span><span class="s2">"reportupdater-cx"</span><span class="w"> </span>/srv/reportupdater/log/reportupdater-cx/syslog.log <span class="p">&amp;</span><span class="w"> </span>stop elukey@stat1006:~$<span class="w"> </span>ls<span class="w"> </span>-l<span class="w"> </span>/srv/reportupdater/log/reportupdater-cx/syslog.log -rw-r--r--<span class="w"> </span><span class="m">1</span><span class="w"> </span>stats<span class="w"> </span>stats<span class="w"> </span><span class="m">647027</span><span class="w"> </span>Mar<span class="w"> </span><span class="m">27</span><span class="w"> </span><span class="m">06</span>:00<span class="w"> </span>/srv/reportupdater/log/reportupdater-cx/syslog.log </pre></div><p>In this case, the journald logs are also replicated to <code>/srv/reportupdater/log/reportupdater-cx/syslog.log</code> </p><div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Failed_service_procedure_(ops_week)" data-mw-thread-id="h-Failed_service_procedure_(ops_week)"><span id="Failed_service_procedure_.28ops_week.29"></span><span data-mw-comment-start="" id="h-Failed_service_procedure_(ops_week)"></span>Failed service procedure (ops week)<span data-mw-comment-end="h-Failed_service_procedure_(ops_week)"></span></h2><!--__DTELLIPSISBUTTON__{"threadItem":{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Failed_service_procedure_(ops_week)","replies":["h-Checking_logs-Failed_service_procedure_(ops_week)","h-Resetting_service_status_to_normal-Failed_service_procedure_(ops_week)"]}}--></div> <p>What to do when it's your ops week and an email has been sent to analytics-alert about a failed-timer? </p> <div class="mw-heading mw-heading3"><h3 id="Checking_logs" data-mw-thread-id="h-Checking_logs-Failed_service_procedure_(ops_week)"><span data-mw-comment-start="" id="h-Checking_logs-Failed_service_procedure_(ops_week)"></span>Checking logs<span data-mw-comment-end="h-Checking_logs-Failed_service_procedure_(ops_week)"></span></h3></div> <p>Depending on jobs, logs are stored only in <code>journald</code> logs (see above), or split between <code>journald</code> and a log-file defined in the job command-line. </p><p>Usefull command: </p><p><code>sudo journalctl -u SERVICE_NAME</code> </p> <div class="mw-heading mw-heading3"><h3 id="Resetting_service_status_to_normal" data-mw-thread-id="h-Resetting_service_status_to_normal-Failed_service_procedure_(ops_week)"><span data-mw-comment-start="" id="h-Resetting_service_status_to_normal-Failed_service_procedure_(ops_week)"></span>Resetting service status to normal<span data-mw-comment-end="h-Resetting_service_status_to_normal-Failed_service_procedure_(ops_week)"></span></h3></div> <p>There are two options to reset a failed service to normal status (after investigation, potential correction etc): </p><p><a href="/wiki/Monitoring/systemd_unit_state" title="Monitoring/systemd unit state">Monitoring/systemd_unit_state</a> </p> <!-- NewPP limit report Parsed by mw‐web.eqiad.main‐958868bfc‐bvtq9 Cached time: 20241104185647 Cache expiry: 2592000 Reduced expiry: false Complications: [show‐toc] DiscussionTools time usage: 0.023 seconds CPU time usage: 0.079 seconds Real time usage: 0.579 seconds Preprocessor visited node count: 173/1000000 Post‐expand include size: 1649/2097152 bytes Template argument size: 1121/2097152 bytes Highest expansion depth: 6/100 Expensive parser function count: 14/500 Unstrip recursion depth: 0/20 Unstrip post‐expand size: 23506/5000000 bytes --> <!-- Transclusion expansion time report (%,ms,calls,template) 100.00% 231.235 1 -total 8.38% 19.389 2 Template:Warn 7.00% 16.181 2 Template:Note --> <!-- Saved in parser cache with key labswiki:pcache:idhash:443309-0!canonical and timestamp 20241104185647 and revision id 2201923. Rendering was triggered because: page-view --> </div><!--esi <esi:include src="/esitest-fa8a495983347898/content" /> --><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://wikitech.wikimedia.org/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;oldid=2201923">https://wikitech.wikimedia.org/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;oldid=2201923</a>"</div></div> <div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Special:Categories" title="Special:Categories">Categories</a>: <ul><li><a href="/wiki/Category:Data_platform" title="Category:Data platform">Data platform</a></li><li><a href="/wiki/Category:Data_platform_systems" title="Category:Data platform systems">Data platform systems</a></li></ul></div></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 1 July 2024, at 20:04.</li> <li id="footer-info-copyright">Text is available under the <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-ShareAlike License</a>; additional terms may apply. See <a class="external text" href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use">Terms of Use</a> for details.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Main_Page">About Wikitech</a></li> <li id="footer-places-disclaimers"><a href="https://foundation.wikimedia.org/wiki/General_disclaimer">Disclaimers</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/wikitech.wikimedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//wikitech.wikimedia.org/w/index.php?title=Data_Platform/Systems/Managing_systemd_timers&amp;mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.canary-84779d6bf6-vqkvc","wgBackendResponseTime":123,"wgDiscussionToolsPageThreads":[{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Anatomy_of_a_Systemd_timer","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-ExecStart_and_difference_with_bash-Anatomy_of_a_Systemd_timer","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Calculating_OnCalendar_interval-Anatomy_of_a_Systemd_timer","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Operations","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-List_all_timers_running_on_the_host-Operations","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-List_command_timers_are_actually_running-Operations","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Check_which_user_runs_a_timer-Operations","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Disable_a_Systemd_timer-Operations","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Shut_off_/_Reset_Icinga_alert_already_fired-Operations","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Check_logs_for_a_systemd_timer-Operations","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Failed_service_procedure_(ops_week)","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Checking_logs-Failed_service_procedure_(ops_week)","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Resetting_service_status_to_normal-Failed_service_procedure_(ops_week)","replies":[]}]}],"wgPageParseReport":{"discussiontools":{"limitreport-timeusage":"0.023"},"limitreport":{"cputime":"0.079","walltime":"0.579","ppvisitednodes":{"value":173,"limit":1000000},"postexpandincludesize":{"value":1649,"limit":2097152},"templateargumentsize":{"value":1121,"limit":2097152},"expansiondepth":{"value":6,"limit":100},"expensivefunctioncount":{"value":14,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":23506,"limit":5000000},"timingprofile":["100.00% 231.235 1 -total"," 8.38% 19.389 2 Template:Warn"," 7.00% 16.181 2 Template:Note"]},"cachereport":{"origin":"mw-web.eqiad.main-958868bfc-bvtq9","timestamp":"20241104185647","ttl":2592000,"transientcontent":false}}});});</script> </body> </html>

Pages: 1 2 3 4 5 6 7 8 9 10